diff --git a/Cargo.toml b/Cargo.toml index 211801b..cc5f88f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,12 +30,13 @@ cpp_portable = ["cpp"] # Optimize FastPFOR for the current CPU. cpp_native = ["cpp"] cpp = ["dep:cmake", "dep:cxx", "dep:cxx-build"] -rust = ["dep:bytes"] +rust = ["dep:bytes", "dep:num-traits"] [dependencies] bytemuck = { version = "1.25.0", features = ["min_const_generics"] } bytes = { version = "1.11", optional = true } cxx = { version = "1.0.194", optional = true } +num-traits = { version = "0.2", optional = true } thiserror = "2.0.18" [build-dependencies] diff --git a/README.md b/README.md index 2309fb8..809d81f 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,34 @@ codec.decode_blocks(&encoded, Some(u32::try_from(blocks.len() * 256).expect("blo assert_eq!(decoded, input); ``` +### 64-bit integers (`u64`) + +The `FastPForWide128` / `FastPForWide256` codecs compress `u64` values. +They implement `AnyLenCodec` (with `Elem = u64`) for native use, and `BlockCodec64` +(`encode64` / `decode64`) for comparison against the C++ codecs. +The wire format is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. + +```rust +use fastpfor::{AnyLenCodec, FastPForWide256}; + +let mut codec = FastPForWide256::default(); +let input: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); + +let mut encoded = Vec::new(); +codec.encode(&input, &mut encoded).unwrap(); + +let mut decoded = Vec::new(); +codec.decode(&encoded, &mut decoded, None).unwrap(); + +assert_eq!(decoded, input); +``` + ### C++ Wrapper (`cpp` feature) Enable the `cpp` feature in `Cargo.toml`: ```toml -fastpfor = { version = "0.1", features = ["cpp"] } +fastpfor = { version = "0.9", features = ["cpp"] } ``` All C++ codecs implement the same `AnyLenCodec` trait (`encode` / `decode`), so @@ -90,16 +112,18 @@ The `FASTPFOR_SIMD_MODE` environment variable (`portable` or `native`) can overr ### Rust (`rust` feature) -Rust block codecs require block-aligned input. `CompositeCodec` chains a block codec with a tail codec (e.g. `VariableByte`) to handle arbitrary-length input. `FastPFor256` and `FastPFor128` are type aliases for such composites. - -| Codec | Description | -|--------------------|--------------------------------------------------------------| -| `FastPFor256` | `CompositeCodec` of `FastPForBlock256` + `VariableByte` | -| `FastPFor128` | `CompositeCodec` of `FastPForBlock128` + `VariableByte` | -| `VariableByte` | Variable-byte encoding, MSB is opposite to protobuf's varint | -| `JustCopy` | No compression; useful as a baseline | -| `FastPForBlock256` | `FastPFor` with 256-element blocks; block-aligned input only | -| `FastPForBlock128` | `FastPFor` with 128-element blocks; block-aligned input only | +Rust block codecs require block-aligned input. `CompositeCodec` chains a block codec with a tail codec (e.g. `VariableByte`) to handle arbitrary-length input. `FastPFor256`/`FastPFor128` (for `u32`) and `FastPForWide256`/`FastPForWide128` (for `u64`) are type aliases for such composites. + +| Codec | Description | +|--------------------|-----------------------------------------------------------------| +| `FastPFor256` | `CompositeCodec` of `FastPForBlock256` + `VariableByte` (`u32`) | +| `FastPFor128` | `CompositeCodec` of `FastPForBlock128` + `VariableByte` (`u32`) | +| `FastPForWide256` | `CompositeCodec` of `FastPForBlockWide256` + `VariableByte` (`u64`) | +| `FastPForWide128` | `CompositeCodec` of `FastPForBlockWide128` + `VariableByte` (`u64`) | +| `VariableByte` | Variable-byte encoding, MSB is opposite to protobuf's varint | +| `JustCopy` | No compression; useful as a baseline | +| `FastPForBlock256` | `FastPFor` with 256-element `u32` blocks; block-aligned input only | +| `FastPForBlock128` | `FastPFor` with 128-element `u32` blocks; block-aligned input only | ### C++ (`cpp` feature) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 69e9628..09bf44d 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -62,3 +62,9 @@ name = "compare_fastpfor_128" path = "fuzz_targets/compare_fastpfor_128.rs" test = false doc = false + +[[bin]] +name = "fastpfor_u64" +path = "fuzz_targets/fastpfor_u64.rs" +test = false +doc = false diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs new file mode 100644 index 0000000..37a2eaf --- /dev/null +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -0,0 +1,59 @@ +#![no_main] + +use fastpfor::cpp::{CppFastPFor128, CppFastPFor256}; +use fastpfor::{BlockCodec64, FastPForWide128, FastPForWide256}; +use libfuzzer_sys::fuzz_target; + +#[derive(arbitrary::Arbitrary, Debug)] +struct Input { + data: Vec, + use_256: bool, +} + +fn check(rust: &mut impl BlockCodec64, cpp: &mut impl BlockCodec64, data: &[u64], name: &str) { + let mut rust_enc = Vec::new(); + rust.encode64(data, &mut rust_enc) + .expect("Rust encode64 failed"); + + let mut cpp_enc = Vec::new(); + cpp.encode64(data, &mut cpp_enc) + .expect("C++ encode64 failed"); + + assert_eq!( + rust_enc, cpp_enc, + "{name}: Rust and C++ encode64 bytes differ" + ); + + let mut rust_dec = Vec::new(); + rust.decode64(&rust_enc, &mut rust_dec) + .expect("Rust decode64 of own output failed"); + assert_eq!(rust_dec, data, "{name}: Rust roundtrip mismatch"); + + let mut cross = Vec::new(); + rust.decode64(&cpp_enc, &mut cross) + .expect("Rust decode64 of C++ output failed"); + assert_eq!(cross, data, "{name}: Rust could not decode C++ output"); + + let mut cpp_dec = Vec::new(); + cpp.decode64(&rust_enc, &mut cpp_dec) + .expect("C++ decode64 of Rust output failed"); + assert_eq!(cpp_dec, data, "{name}: C++ could not decode Rust output"); +} + +fuzz_target!(|input: Input| { + if input.use_256 { + check( + &mut FastPForWide256::default(), + &mut CppFastPFor256::default(), + &input.data, + "FastPForWide256", + ); + } else { + check( + &mut FastPForWide128::default(), + &mut CppFastPFor128::default(), + &input.data, + "FastPForWide128", + ); + } +}); diff --git a/src/codec.rs b/src/codec.rs index 9f55563..087eddb 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -28,6 +28,7 @@ pub(crate) fn default_max_decoded_len(compressed_words: usize) -> usize { /// #[derive(Default)] /// struct MyCodec; /// impl BlockCodec for MyCodec { +/// type Elem = u32; /// type Block = [u32; 256]; /// fn encode_blocks(&mut self, blocks: &[[u32; 256]], out: &mut Vec) /// -> FastPForResult<()> { todo!() } @@ -36,21 +37,25 @@ pub(crate) fn default_max_decoded_len(compressed_words: usize) -> usize { /// } /// ``` pub trait BlockCodec: Default { - /// The fixed-size block type. Must be plain-old-data (`Pod`). - /// In practice this will be `[u32; 128]` or `[u32; 256]`. + /// The unpacked element type: [`u32`] or [`u64`]. + /// The compressed stream is always `Vec`; only decoded values use this width. + type Elem: Pod; + + /// The fixed-size block type, which must be plain-old-data (`Pod`). + /// In practice `[u32; 128]`, `[u32; 256]`, `[u64; 128]`, or `[u64; 256]`. type Block: Pod; - /// Number of `u32` elements in one block. + /// Number of [`Elem`](BlockCodec::Elem) values in one block. /// - /// Equal to `size_of::() / 4`. Use this when computing - /// element counts from block counts, e.g. `n_blocks * codec.elements_per_block()`. + /// Equal to `size_of::() / size_of::()`. + /// Use this when computing element counts from block counts. #[inline] #[must_use] fn size() -> usize where Self: Sized, { - size_of::() / size_of::() + size_of::() / size_of::() } /// Compress a slice of complete, fixed-size blocks. @@ -76,7 +81,7 @@ pub trait BlockCodec: Default { &mut self, input: &[u32], expected_len: Option, - out: &mut Vec, + out: &mut Vec, ) -> FastPForResult; /// Maximum decompressed element count for a given compressed input length. @@ -93,13 +98,13 @@ pub trait BlockCodec: Default { /// Codec that supports compressing 64-bit integers into a 32-bit word stream. /// -/// Only three C++ codecs implement this trait: `CppFastPFor128`, -/// `CppFastPFor256`, and `CppVarInt`. For simple use, call -/// `encode64` / `decode64` directly on the struct — no trait import required. +/// Implemented by the pure-Rust [`FastPForWide128`](crate::FastPForWide128) and +/// [`FastPForWide256`](crate::FastPForWide256) codecs. +/// With the `cpp` feature, `CppFastPFor128`, `CppFastPFor256`, and `CppVarInt` also implement it. /// -/// Import `BlockCodec64` only when writing generic code over multiple codecs -/// that support 64-bit compression. -#[cfg(feature = "cpp")] +/// This is a shared interface for cross-codec comparison; for native Rust use, +/// [`FastPForWide128`](crate::FastPForWide128) also implements [`AnyLenCodec`] with `Elem = u64`. +/// Import `BlockCodec64` only when writing generic code over several 64-bit codecs. pub trait BlockCodec64 { /// Compress 64-bit integers into a 32-bit word stream. fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()>; @@ -114,8 +119,12 @@ pub trait BlockCodec64 { /// trait directly. Block-oriented codecs are wrapped in `CompositeCodec` /// to produce an `AnyLenCodec`. pub trait AnyLenCodec: Default { - /// Compress an arbitrary-length slice of `u32` values. - fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()>; + /// The unpacked element type: [`u32`] or [`u64`]. + /// The compressed stream is always `Vec`; only decoded values use this width. + type Elem; + + /// Compress an arbitrary-length slice of [`Elem`](AnyLenCodec::Elem) values. + fn encode(&mut self, input: &[Self::Elem], out: &mut Vec) -> FastPForResult<()>; /// Maximum decompressed element count for a given compressed input length. /// Reject `expected_len` values exceeding this to avoid allocation from bad data. @@ -140,7 +149,7 @@ pub trait AnyLenCodec: Default { fn decode( &mut self, input: &[u32], - out: &mut Vec, + out: &mut Vec, expected_len: Option, ) -> FastPForResult<()>; } @@ -164,9 +173,11 @@ pub trait AnyLenCodec: Default { /// assert_eq!(remainder.len(), 88); /// ``` #[must_use] -pub fn slice_to_blocks(input: &[u32]) -> (&[Blocks::Block], &[u32]) { - let block_u32s = Blocks::size(); - let aligned_down = (input.len() / block_u32s) * block_u32s; +pub fn slice_to_blocks( + input: &[Blocks::Elem], +) -> (&[Blocks::Block], &[Blocks::Elem]) { + let block_elems = Blocks::size(); + let aligned_down = (input.len() / block_elems) * block_elems; let (aligned, remainder) = input.split_at(aligned_down); let blocks: &[Blocks::Block] = cast_slice(aligned); // must not panic (blocks, remainder) diff --git a/src/cpp/codecs.rs b/src/cpp/codecs.rs index be18dcd..24bf91b 100644 --- a/src/cpp/codecs.rs +++ b/src/cpp/codecs.rs @@ -37,6 +37,8 @@ macro_rules! implement_cpp_codecs { } impl AnyLenCodec for $name { + type Elem = u32; + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { encode32_to_vec_ffi(&self.0, input, out) } diff --git a/src/helpers.rs b/src/helpers.rs index cf0d438..d61f276 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -6,13 +6,6 @@ pub fn greatest_multiple(value: u32, factor: u32) -> u32 { value - value % factor } -/// Returns the number of bits needed to represent `i`. -/// Returns 0 for input 0. -#[cfg_attr(feature = "cpp", allow(dead_code))] -pub fn bits(i: u32) -> usize { - 32 - i.leading_zeros().as_usize() -} - pub trait AsUsize: Eq + Copy { fn as_usize(self) -> usize; diff --git a/src/lib.rs b/src/lib.rs index a62c545..b7bd39e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,9 +18,7 @@ pub mod cpp; pub(crate) mod rust; mod codec; -#[cfg(feature = "cpp")] -pub use codec::BlockCodec64; -pub use codec::{AnyLenCodec, BlockCodec, slice_to_blocks}; +pub use codec::{AnyLenCodec, BlockCodec, BlockCodec64, slice_to_blocks}; pub(crate) mod helpers; @@ -31,7 +29,8 @@ pub use bytemuck::Pod; #[cfg(feature = "rust")] pub use rust::{ CompositeCodec, FastPFor, FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, - JustCopy, VariableByte, + FastPForBlockWide128, FastPForBlockWide256, FastPForWide128, FastPForWide256, JustCopy, + VariableByte, }; // `src/test_utils.rs` uses `fastpfor::...`; alias this crate for unit tests only. diff --git a/src/rust/composite.rs b/src/rust/composite.rs index adea3c1..2501d69 100644 --- a/src/rust/composite.rs +++ b/src/rust/composite.rs @@ -40,26 +40,33 @@ use crate::helpers::AsUsize; /// codec.decode(&encoded, &mut decoded, None).unwrap(); /// assert_eq!(decoded, data); /// ``` -pub struct CompositeCodec { +#[derive(Debug)] +pub struct CompositeCodec> { block: Blocks, tail: Tail, } -impl Default for CompositeCodec { +impl> Default + for CompositeCodec +{ fn default() -> Self { Self::new(Blocks::default(), Tail::default()) } } -impl CompositeCodec { +impl> CompositeCodec { /// Creates a new `CompositeCodec` from a block codec and a tail codec. pub fn new(block: Blocks, tail: Tail) -> Self { Self { block, tail } } } -impl AnyLenCodec for CompositeCodec { - fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { +impl> AnyLenCodec + for CompositeCodec +{ + type Elem = Blocks::Elem; + + fn encode(&mut self, input: &[Self::Elem], out: &mut Vec) -> FastPForResult<()> { let (blocks, remainder) = slice_to_blocks::(input); // C++ CompositeCodec: concatenate block + tail. Block codec writes length header (0 when empty). self.block.encode_blocks(blocks, out)?; @@ -70,7 +77,7 @@ impl AnyLenCodec for CompositeCodec, + out: &mut Vec, expected_len: Option, ) -> FastPForResult<()> { let start_len = out.len(); diff --git a/src/rust/fastpfor_codec.rs b/src/rust/fastpfor_codec.rs new file mode 100644 index 0000000..d2feb2f --- /dev/null +++ b/src/rust/fastpfor_codec.rs @@ -0,0 +1,114 @@ +//! Public any-length `FastPFOR` codecs. +//! +//! [`FastPFor128`]/[`FastPFor256`] compress `u32`; [`FastPForWide128`]/[`FastPForWide256`] compress `u64`. +//! Each is one [`CompositeCodec`]: the width-generic block engine plus a variable-byte tail for the remainder. + +use crate::FastPForResult; +use crate::codec::{AnyLenCodec, BlockCodec64}; +use crate::rust::VariableByte; +use crate::rust::composite::CompositeCodec; +use crate::rust::integer_compression::fastpfor::{FastPFor, sealed}; +use crate::rust::integer_compression::fastpfor_int::FastPForInt; + +/// Any-length `FastPFOR` codec over `N`-value blocks of width `T` ([`u32`] or [`u64`]). +/// +/// A single [`CompositeCodec`] pairing the width-generic block engine with a [`VariableByte`] tail. +/// Instantiate through the [`FastPFor128`]/[`FastPForWide128`] aliases. +#[derive(Debug)] +pub struct FastPForCodec +where + [T; N]: sealed::BlockSize, + VariableByte: AnyLenCodec, +{ + inner: CompositeCodec, VariableByte>, +} + +// Hand-written (not derived) so `default()` needs no `T: Default` bound; +// the tail's `AnyLenCodec: Default` supertrait already guarantees it. +impl Default for FastPForCodec +where + [T; N]: sealed::BlockSize, + VariableByte: AnyLenCodec, +{ + fn default() -> Self { + Self { + inner: CompositeCodec::default(), + } + } +} + +impl AnyLenCodec for FastPForCodec +where + [T; N]: sealed::BlockSize, + VariableByte: AnyLenCodec, +{ + type Elem = T; + + fn encode(&mut self, input: &[T], out: &mut Vec) -> FastPForResult<()> { + self.inner.encode(input, out) + } + + fn decode( + &mut self, + input: &[u32], + out: &mut Vec, + expected_len: Option, + ) -> FastPForResult<()> { + self.inner.decode(input, out, expected_len) + } +} + +/// Compresses 64-bit integers through the shared [`BlockCodec64`] interface. +/// +/// Lets the `u64` codecs be compared against the C++ codecs, which expose `u64` the same way. +impl BlockCodec64 for FastPForCodec +where + [u64; N]: sealed::BlockSize, +{ + fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { + self.inner.encode(input, out) + } + + fn decode64(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { + self.inner.decode(input, out, None) + } +} + +/// Any-length `u32` `FastPFOR` codec with 128-value blocks. +pub type FastPFor128 = FastPForCodec<128, u32>; + +/// Any-length `u32` `FastPFOR` codec with 256-value blocks. +pub type FastPFor256 = FastPForCodec<256, u32>; + +/// Any-length `u64` `FastPFOR` codec with 128-value blocks. +pub type FastPForWide128 = FastPForCodec<128, u64>; + +/// Any-length `u64` `FastPFOR` codec with 256-value blocks. +pub type FastPForWide256 = FastPForCodec<256, u64>; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn narrow_codec_roundtrips_u32() { + let mut codec = FastPFor256::default(); + let data: Vec = (0..600).collect(); + let mut enc = Vec::new(); + codec.encode(&data, &mut enc).unwrap(); + let mut dec = Vec::new(); + codec.decode(&enc, &mut dec, None).unwrap(); + assert_eq!(dec, data); + } + + #[test] + fn wide_codec_roundtrips_u64() { + let mut codec = FastPForWide256::default(); + let data: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); + let mut enc = Vec::new(); + codec.encode(&data, &mut enc).unwrap(); + let mut dec = Vec::new(); + codec.decode(&enc, &mut dec, None).unwrap(); + assert_eq!(dec, data); + } +} diff --git a/src/rust/integer_compression/bitpacking.rs b/src/rust/integer_compression/bit_pack32.rs similarity index 99% rename from src/rust/integer_compression/bitpacking.rs rename to src/rust/integer_compression/bit_pack32.rs index d911113..8a7b839 100644 --- a/src/rust/integer_compression/bitpacking.rs +++ b/src/rust/integer_compression/bit_pack32.rs @@ -1305,7 +1305,7 @@ mod tests { use rand::RngExt as _; use super::fast_pack; - use crate::rust::integer_compression::bitunpacking::fast_unpack; + use crate::rust::integer_compression::bit_unpack32::fast_unpack; #[test] fn pack_unpack_roundtrip() { diff --git a/src/rust/integer_compression/bit_pack64.rs b/src/rust/integer_compression/bit_pack64.rs new file mode 100644 index 0000000..597785c --- /dev/null +++ b/src/rust/integer_compression/bit_pack64.rs @@ -0,0 +1,116 @@ +//! Generic scalar bit-packing for 64-bit values. +//! +//! Packs and unpacks groups of 32 values at any bit width `0..=64`. +//! The layout is a little-endian bitstream, matching the hand-unrolled 32-bit kernels in [`bitpacking`](super::bit_pack32). +//! Value `j` occupies bits `[j*bit, (j+1)*bit)` of the concatenated stream. +//! Each call moves exactly `bit` `u32` words. + +const fn low_mask(bit: u8) -> u64 { + if bit >= 64 { + u64::MAX + } else { + (1u64 << bit) - 1 + } +} + +/// Packs 32 values from `input[inpos..]` into `output[outpos..]` at `bit` bits each. +pub fn pack_wide(input: &[u64], inpos: usize, output: &mut [u32], outpos: usize, bit: u8) { + if bit == 0 { + return; + } + let mask = u128::from(low_mask(bit)); + let mut acc: u128 = 0; + let mut filled: u32 = 0; + let mut out = outpos; + for j in 0..32 { + acc |= (u128::from(input[inpos + j]) & mask) << filled; + filled += u32::from(bit); + while filled >= 32 { + output[out] = acc as u32; + out += 1; + acc >>= 32; + filled -= 32; + } + } +} + +/// Unpacks 32 values from `input[inpos..]` into `output[outpos..]` at `bit` bits each. +pub fn unpack_wide(input: &[u32], inpos: usize, output: &mut [u64], outpos: usize, bit: u8) { + if bit == 0 { + output[outpos..outpos + 32].fill(0); + return; + } + let mask = u128::from(low_mask(bit)); + let mut acc: u128 = 0; + let mut avail: u32 = 0; + let mut inp = inpos; + for j in 0..32 { + while avail < u32::from(bit) { + acc |= u128::from(input[inp]) << avail; + inp += 1; + avail += 32; + } + output[outpos + j] = (acc & mask) as u64; + acc >>= u32::from(bit); + avail -= u32::from(bit); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rust::integer_compression::{bit_pack32, bit_unpack32}; + + #[test] + fn wide_matches_u32_kernels() { + let values32: [u32; 32] = std::array::from_fn(|i| (i as u32).wrapping_mul(2_654_435_761)); + + for bit in 1..=32u8 { + let mask = if bit == 32 { + u32::MAX + } else { + (1u32 << bit) - 1 + }; + let masked32: [u32; 32] = std::array::from_fn(|i| values32[i] & mask); + let masked64: [u64; 32] = std::array::from_fn(|i| u64::from(masked32[i])); + + let mut out_ref = vec![0u32; bit as usize]; + bit_pack32::fast_pack(&masked32, 0, &mut out_ref, 0, bit); + + let mut out_wide = vec![0u32; bit as usize]; + pack_wide(&masked64, 0, &mut out_wide, 0, bit); + + assert_eq!(out_ref, out_wide, "pack mismatch at bit={bit}"); + + let mut back_ref = vec![0u32; 32]; + bit_unpack32::fast_unpack(&out_ref, 0, &mut back_ref, 0, bit); + let mut back_wide = vec![0u64; 32]; + unpack_wide(&out_wide, 0, &mut back_wide, 0, bit); + + for i in 0..32 { + assert_eq!( + u64::from(back_ref[i]), + back_wide[i], + "unpack mismatch at bit={bit}" + ); + } + } + } + + #[test] + fn wide_roundtrip_all_widths() { + for bit in 0..=64u8 { + let mask = low_mask(bit); + let values: [u64; 32] = + std::array::from_fn(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) & mask); + + let mut packed = vec![0u32; bit as usize]; + pack_wide(&values, 0, &mut packed, 0, bit); + + let mut back = vec![0u64; 32]; + unpack_wide(&packed, 0, &mut back, 0, bit); + + assert_eq!(values.to_vec(), back, "roundtrip mismatch at bit={bit}"); + } + } +} diff --git a/src/rust/integer_compression/bitunpacking.rs b/src/rust/integer_compression/bit_unpack32.rs similarity index 100% rename from src/rust/integer_compression/bitunpacking.rs rename to src/rust/integer_compression/bit_unpack32.rs diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index fa6f6bc..43b69ec 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -1,23 +1,24 @@ -use std::array; use std::cmp::min; use std::io::Cursor; use bytemuck::cast_slice; use bytes::{Buf as _, BufMut as _, BytesMut}; -use crate::helpers::{AsUsize, GetWithErr, bits, greatest_multiple}; +use crate::helpers::{GetWithErr, greatest_multiple}; use crate::rust::cursor::IncrementCursor; -use crate::rust::integer_compression::{bitpacking, bitunpacking}; -use crate::{BlockCodec, FastPForError, FastPForResult}; +use crate::rust::integer_compression::fastpfor_int::FastPForInt; +use crate::{FastPForError, FastPForResult}; -mod sealed { - /// Sealed marker trait: only `[u32; 128]` and `[u32; 256]` are valid `FastPFor` block arrays. +pub(crate) mod sealed { + /// Sealed marker trait: only valid `[T; N]` block arrays are accepted by `FastPFor`. /// /// This is intentionally private so that users cannot implement it for other sizes, - /// preventing instantiation of `FastPFor` for unsupported `N` at compile time. + /// preventing instantiation of `FastPFor` for unsupported `N`/`T` at compile time. pub trait BlockSize: bytemuck::Pod {} impl BlockSize for [u32; 128] {} impl BlockSize for [u32; 256] {} + impl BlockSize for [u64; 128] {} + impl BlockSize for [u64; 256] {} } /// Overhead cost (in bits) for storing each exception's position in the block @@ -26,19 +27,19 @@ const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; /// Default page size in number of integers. const DEFAULT_PAGE_SIZE: u32 = 65536; -/// Type alias for [`FastPFor`] with 128-element blocks. -pub type FastPForBlock128 = FastPFor<128>; - -/// Type alias for [`FastPFor`] with 256-element blocks. -pub type FastPForBlock256 = FastPFor<256>; - /// Fast Patched Frame-of-Reference ([FastPFOR](https://github.com/lemire/FastPFor)) codec. /// -/// `N` is the block size (128 or 256 values per block). This struct implements -/// [`BlockCodec`] with `Block = [u32; N]`, giving compile-time guarantees that -/// only correctly-sized blocks are accepted. +/// `N` is the block size (128 or 256 values per block) and `T` the element type +/// ([`u32`] or [`u64`], defaulting to `u32`). This struct implements [`BlockCodec`](crate::BlockCodec) +/// with `Block = [u32; N]` for the `u32` element type, giving compile-time guarantees that only +/// correctly-sized blocks are accepted. +/// +/// The per-block scratch buffers are sized exactly for `T` (`T::WIDTH + 1` buckets) via the +/// sealed `FastPForInt` trait, so the bucket count is neither wasted nor part of this +/// type's signature. /// -/// Use [`FastPForBlock128`] or [`FastPForBlock256`] as convenient type aliases. +/// Use [`FastPForBlock128`](crate::FastPForBlock128) or [`FastPForBlock256`](crate::FastPForBlock256) +/// as convenient `u32` type aliases. /// /// To compress arbitrary-length data (including a sub-block remainder), /// wrap this in a [`CompositeCodec`](crate::CompositeCodec): @@ -51,18 +52,18 @@ pub type FastPForBlock256 = FastPFor<256>; /// codec.encode(&data, &mut out).unwrap(); /// ``` #[derive(Debug)] -pub struct FastPFor { +pub struct FastPFor { /// Exception values indexed by bit width difference - exception_buffers: [Vec; 33], + exception_buffers: T::ExceptionBuffers, /// Metadata buffer for encoding/decoding bytes_container: BytesMut, /// Maximum integers per page page_size: u32, /// Position trackers for exception arrays - data_pointers: [usize; 33], + data_pointers: T::DataPointers, /// Frequency count for each bit width: /// `freqs[i]` = count of values needing exactly i bits - freqs: [u32; 33], + freqs: T::Freqs, /// Optimal number of bits chosen for the current block optimal_bits: u8, /// Number of exceptions that don't fit in the optimal bit width @@ -71,20 +72,17 @@ pub struct FastPFor { max_bits: u8, } -impl Default for FastPFor -where - [u32; N]: sealed::BlockSize, -{ +impl Default for FastPFor { fn default() -> Self { Self::new(DEFAULT_PAGE_SIZE) .expect("DEFAULT_PAGE_SIZE is a multiple of all valid block sizes") } } -impl FastPFor { +impl FastPFor { /// Creates a new `FastPForBlock` with a codec with the given page size. /// - /// Returns an error if `page_size` is not a multiple of 128. + /// Returns an error if `page_size` is not a multiple of the block size. /// Use [`Default`] for the default page size. pub fn new(page_size: u32) -> FastPForResult { if page_size % N as u32 != 0 { @@ -98,18 +96,18 @@ impl FastPFor { (3 * page_size / N as u32 + page_size) as usize, ), page_size, - exception_buffers: array::from_fn(|_| Vec::new()), - data_pointers: [0; 33], - freqs: [0; 33], + exception_buffers: T::new_exception_buffers(), + data_pointers: T::new_data_pointers(), + freqs: T::new_freqs(), optimal_bits: 0, exception_count: 0, max_bits: 0, }) } - fn compress_blocks( + pub(crate) fn compress_blocks( &mut self, - input: &[u32], + input: &[T], input_length: u32, input_offset: &mut Cursor, output: &mut [u32], @@ -123,12 +121,12 @@ impl FastPFor { } } - fn decode_headless_blocks( + pub(crate) fn decode_headless_blocks( &mut self, input: &[u32], inlength: u32, input_offset: &mut Cursor, - output: &mut [u32], + output: &mut [T], output_offset: &mut Cursor, ) -> FastPForResult<()> { let mynvalue = greatest_multiple(inlength, N as u32); @@ -153,7 +151,7 @@ impl FastPFor { /// * `output_offset` - Advanced by compressed size fn encode_page( &mut self, - input: &[u32], + input: &[T], this_size: u32, input_offset: &mut Cursor, output: &mut [u32], @@ -164,7 +162,7 @@ impl FastPFor { let mut tmp_output_offset = output_offset.position() as u32; // Data pointers to 0 - self.data_pointers.fill(0); + self.data_pointers.as_mut().fill(0); self.bytes_container.clear(); let mut tmp_input_offset = input_offset.position() as u32; @@ -180,19 +178,22 @@ impl FastPFor { if needed > self.exception_buffers[index].len() { // Grow to the next multiple of 32 above 2×needed, to amortize resizes. let new_cap = needed.saturating_mul(2).next_multiple_of(32); - self.exception_buffers[index].resize(new_cap, 0); + self.exception_buffers[index].resize(new_cap, T::zero()); } for k in 0..N as u32 { - if (input[(k + tmp_input_offset) as usize] >> self.optimal_bits) != 0 { + if input[(k + tmp_input_offset) as usize] >> usize::from(self.optimal_bits) + != T::zero() + { self.bytes_container.put_u8(k as u8); - self.exception_buffers[index][self.data_pointers[index]] = - input[(k + tmp_input_offset) as usize] >> self.optimal_bits; + self.exception_buffers[index][self.data_pointers[index]] = input + [(k + tmp_input_offset) as usize] + >> usize::from(self.optimal_bits); self.data_pointers[index] += 1; } } } for k in (0..N as u32).step_by(32) { - bitpacking::fast_pack( + T::fast_pack( input, (tmp_input_offset + k) as usize, output, @@ -218,22 +219,23 @@ impl FastPFor { output[tmp_output_offset as usize..][..how_many_ints] .copy_from_slice(&meta_u32s[..how_many_ints]); tmp_output_offset += how_many_ints as u32; - let mut bitmap = 0; - for k in 2..=32 { + // Exception bitmap: one bit per bit-width bucket, written as `T::BITMAP_WORDS` words. + let mut bitmap = T::zero(); + for k in 2..=usize::from(T::WIDTH) { if self.data_pointers[k] != 0 { - bitmap |= 1 << (k - 1); + bitmap |= T::one() << (k - 1); } } - output[tmp_output_offset as usize] = bitmap; - tmp_output_offset += 1; + T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); + tmp_output_offset += T::BITMAP_WORDS; - for k in 2..=32 { + for k in 2..=usize::from(T::WIDTH) { if self.data_pointers[k] != 0 { output[tmp_output_offset as usize] = self.data_pointers[k] as u32; tmp_output_offset += 1; let mut j = 0; while j < self.data_pointers[k] { - bitpacking::fast_pack( + T::fast_pack( &self.exception_buffers[k], j, output, @@ -255,14 +257,14 @@ impl FastPFor { /// Computes optimal bit width minimizing total storage cost. /// /// Analyzes frequency distribution to balance regular value bits against exception overhead. - fn best_bit_from_data(&mut self, input: &[u32], pos: u32) { - self.freqs.fill(0); + fn best_bit_from_data(&mut self, input: &[T], pos: u32) { + self.freqs.as_mut().fill(0); let k_end = min(pos + N as u32, input.len() as u32); for k in pos..k_end { - self.freqs[bits(input[k as usize])] += 1; + self.freqs[usize::from(input[k as usize].significant_bits())] += 1; } - self.optimal_bits = 32; + self.optimal_bits = T::WIDTH; while self.freqs[self.optimal_bits as usize] == 0 { self.optimal_bits -= 1; } @@ -307,7 +309,7 @@ impl FastPFor { &mut self, input: &[u32], input_offset: &mut Cursor, - output: &mut [u32], + output: &mut [T], output_offset: &mut Cursor, this_size: u32, ) -> FastPForResult<()> { @@ -340,13 +342,13 @@ impl FastPFor { .checked_add(length) .ok_or(FastPForError::NotEnoughData)?; - let bitmap = input.get_val(inexcept)?; + let bitmap = T::read_bitmap(input, inexcept)?; inexcept = inexcept - .checked_add(1) + .checked_add(T::BITMAP_WORDS) .ok_or(FastPForError::NotEnoughData)?; - for k in 2..=32 { - if (bitmap & (1 << (k - 1))) != 0 { + for k in 2..=u32::from(T::WIDTH) { + if bitmap & (T::one() << (k - 1) as usize) != T::zero() { let size = input.get_val(inexcept)?; inexcept = inexcept .checked_add(1) @@ -359,14 +361,14 @@ impl FastPFor { // to the next group of 32 for the bitunpacking calls. let rounded_up = size.next_multiple_of(32) as usize; if self.exception_buffers[k as usize].len() < rounded_up { - self.exception_buffers[k as usize].resize(rounded_up, 0); + self.exception_buffers[k as usize].resize(rounded_up, T::zero()); } let mut j: u32 = 0; // Process full groups directly from input while j.checked_add(32).is_some_and(|j32| j32 <= size) && inexcept.checked_add(k).is_some_and(|ie| ie <= n) { - bitunpacking::fast_unpack( + T::fast_unpack( input, inexcept as usize, &mut self.exception_buffers[k as usize], @@ -396,7 +398,7 @@ impl FastPFor { .ok_or(FastPForError::NotEnoughData)?; tail_buf[..copy_len].copy_from_slice(src); let tail_inpos = 0; - bitunpacking::fast_unpack( + T::fast_unpack( &tail_buf, tail_inpos, &mut self.exception_buffers[k as usize], @@ -411,14 +413,14 @@ impl FastPFor { } } - self.data_pointers.fill(0); + self.data_pointers.as_mut().fill(0); let mut tmp_output_offset = output_offset.position() as u32; let mut tmp_input_offset = input_offset.position() as u32; let run_end = this_size / N as u32; for _ in 0..run_end { let bits = input_bytes.get_val(byte_pos)?; - if bits > 32 { + if bits > T::WIDTH { return Err(FastPForError::NotEnoughData); } byte_pos += 1; @@ -439,7 +441,7 @@ impl FastPFor { if out_end > output.len() { return Err(FastPForError::OutputBufferTooSmall); } - bitunpacking::fast_unpack(input, in_start, output, out_start, bits); + T::fast_unpack(input, in_start, output, out_start, bits); tmp_input_offset += u32::from(bits); } if num_exceptions > 0 { @@ -448,7 +450,7 @@ impl FastPFor { let index = maxbits .checked_sub(bits) .ok_or(FastPForError::NotEnoughData)?; - if maxbits > 32 || index == 0 || index > 32 { + if maxbits > T::WIDTH || index == 0 || index > T::WIDTH { return Err(FastPForError::NotEnoughData); } let index = usize::from(index); @@ -463,7 +465,7 @@ impl FastPFor { if out_idx >= output.len() { return Err(FastPForError::OutputBufferTooSmall); } - output[out_idx] |= 1 << bits; + output[out_idx] |= T::one() << usize::from(bits); } } else { for _ in 0..num_exceptions { @@ -478,7 +480,7 @@ impl FastPFor { } let ptr = self.data_pointers[index]; let except_value = self.exception_buffers[index].get_val(ptr)?; - output[out_idx] |= except_value << bits; + output[out_idx] |= except_value << usize::from(bits); self.data_pointers[index] += 1; } } @@ -490,225 +492,3 @@ impl FastPFor { Ok(()) } } - -impl BlockCodec for FastPFor -where - [u32; N]: sealed::BlockSize, -{ - type Block = [u32; N]; - - fn encode_blocks(&mut self, blocks: &[Self::Block], out: &mut Vec) -> FastPForResult<()> { - let n_values = (blocks.len() * N) as u32; - if blocks.is_empty() { - out.push(n_values); - return Ok(()); - } - let flat: &[u32] = cast_slice(blocks); - - let capacity = flat.len() * 2 + 1024; - let start = out.len(); - // Reserve slot for the length header, then space for compressed data. - out.resize(start + 1 + capacity, 0); - - let mut in_off = Cursor::new(0u32); - let mut out_off = Cursor::new(0u32); - - // Write length header then compress. - out[start] = n_values; - self.compress_blocks( - flat, - n_values, - &mut in_off, - &mut out[start + 1..], - &mut out_off, - ); - - let written = 1 + out_off.position() as usize; - out.truncate(start + written); - Ok(()) - } - - fn decode_blocks( - &mut self, - input: &[u32], - expected_len: Option, - out: &mut Vec, - ) -> FastPForResult { - let Some((&block_n_values, rest)) = input.split_first() else { - return Err(FastPForError::NotEnoughData); - }; - if block_n_values % N as u32 != 0 { - return Err(FastPForError::NotEnoughData); - } - if let Some(expected) = expected_len { - if block_n_values != expected { - return Err(FastPForError::DecodedCountMismatch { - actual: block_n_values.as_usize(), - expected: expected.as_usize(), - }); - } - } else { - let max = Self::max_decompressed_len(input.len()); - if block_n_values.as_usize() > max { - return Err(FastPForError::NotEnoughData); - } - } - let n_blocks = block_n_values as usize / N; - if n_blocks == 0 { - return Ok(1); - } - let start = out.len(); - out.resize(start + n_blocks * N, 0); - - let mut in_off = Cursor::new(0u32); - let mut out_off = Cursor::new(0u32); - - self.decode_headless_blocks( - rest, - block_n_values, - &mut in_off, - &mut out[start..], - &mut out_off, - )?; - - let written = out_off.position() as usize; - if written != n_blocks * N { - out.truncate(start + written); - } - // +1 for the header word (block_n_values) that precedes `rest`. - Ok(1 + in_off.position() as usize) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{block_compress, block_decompress, block_roundtrip}; - - #[test] - fn fastpfor_test() { - let mut data = vec![0u32; 256]; - data[126] = u32::MAX; - block_roundtrip::(&data); - } - - #[test] - fn fastpfor_test_128() { - let mut data = vec![0u32; 128]; - data[126] = u32::MAX; - block_roundtrip::(&data); - } - - #[test] - fn test_empty_blocks_ok() { - // Empty input encodes to length header [0] (matches C++ FastPFor) and decodes cleanly. - let enc = block_compress::(&[]).unwrap(); - assert_eq!(enc, [0]); - let dec = block_decompress::(&enc, Some(0)).unwrap(); - assert!(dec.is_empty()); - } - - // Tests ported from C++ - #[test] - fn test_constant_sequence() { - block_roundtrip::(&vec![42u32; 65536]); - } - - #[test] - fn test_alternating_sequence() { - let data: Vec<_> = (0..65536u32).map(|i| u32::from(i % 2 != 0)).collect(); - block_roundtrip::(&data); - } - - #[test] - fn test_large_numbers() { - let data: Vec = (0..65536u32).map(|i| i + (1u32 << 30)).collect(); - block_roundtrip::(&data); - } - - #[test] - fn cursor_api_roundtrip() { - block_roundtrip::(&vec![42u32; 256]); - } - - #[test] - fn headless_compress_unfit_pagesize() { - // 640 values with 128-block codec spans two pages (512 + 128), exercising the loop. - let input: Vec = (0..640u32).collect(); - block_roundtrip::(&input); - } - - #[test] - fn exception_value_vector_resizes() { - // Alternating large/small values trigger exception-buffer resizing across pages. - let input: Vec = (0..1024u32) - .map(|i| if i % 2 == 0 { 1 << 30 } else { 3 }) - .collect(); - block_roundtrip::(&input); - } - - // ── Error / edge tests not covered by `tests/decode_validation.rs` ───── - // - // `AnyLenCodec::decode` treats an empty slice as tail-only and succeeds; an empty - // `decode_blocks` input is still invalid. Headless decode is internal-only. - - #[test] - fn uncompress_zero_input_length_err() { - // Truly empty input (no header word at all) is invalid — C++ would crash reading *in. - block_decompress::(&[], None).unwrap_err(); - } - - #[test] - fn headless_uncompress_zero_inlength_128_ok() { - FastPForBlock128::default() - .decode_headless_blocks( - &[], - 0, - &mut Cursor::new(0u32), - &mut [], - &mut Cursor::new(0u32), - ) - .expect("zero-length decompress must succeed"); - } - - #[test] - fn decode_where_meta_overflow() { - // `decode_headless_blocks` only: no `AnyLenCodec` entry point passes this layout. - let data: Vec = (0..256u32) - .map(|i| if i % 2 == 0 { 1u32 << 30 } else { 3 }) - .collect(); - let compressed = block_compress::(&data).unwrap(); - - let mut padded = vec![0u32]; - padded.extend_from_slice(&compressed); - padded[2] = u32::MAX; - let out_length = padded[1]; - assert!( - FastPForBlock256::default() - .decode_headless_blocks( - &padded, - out_length, - &mut Cursor::new(1u32), - &mut vec![0u32; 320], - &mut Cursor::new(0u32), - ) - .is_err() - ); - } - - #[test] - fn decode_index1_branch_valid() { - let mut data = vec![1u32; 256]; - data[0] = 3; - block_roundtrip::(&data); - } - - /// `decode_blocks` with `expected_len: None` and header=0 returns `Ok` with empty output. - #[test] - fn decode_blocks_header_only_input() { - // Input with just the length header [0]: no blocks to decode. - let input = vec![0u32]; - let out = block_decompress::(&input, None).unwrap(); - assert!(out.is_empty()); - } -} diff --git a/src/rust/integer_compression/fastpfor32.rs b/src/rust/integer_compression/fastpfor32.rs new file mode 100644 index 0000000..c2f1315 --- /dev/null +++ b/src/rust/integer_compression/fastpfor32.rs @@ -0,0 +1,237 @@ +use std::io::Cursor; + +use bytemuck::cast_slice; + +use crate::helpers::AsUsize; +use crate::rust::integer_compression::fastpfor::sealed; +use crate::rust::integer_compression::fastpfor_int::FastPForInt; +use crate::{BlockCodec, FastPFor, FastPForError, FastPForResult}; + +/// Type alias for [`FastPFor`] with 128-element `u32` blocks. +pub type FastPForBlock128 = FastPFor<128, u32>; + +/// Type alias for [`FastPFor`] with 256-element `u32` blocks. +pub type FastPForBlock256 = FastPFor<256, u32>; + +impl BlockCodec for FastPFor +where + [T; N]: sealed::BlockSize, +{ + type Elem = T; + type Block = [T; N]; + + fn encode_blocks(&mut self, blocks: &[Self::Block], out: &mut Vec) -> FastPForResult<()> { + let n_values = (blocks.len() * N) as u32; + if blocks.is_empty() { + out.push(n_values); + return Ok(()); + } + let flat: &[T] = cast_slice(blocks); + + let capacity = flat.len() * 3 + 1024; + let start = out.len(); + // Reserve slot for the length header, then space for compressed data. + out.resize(start + 1 + capacity, 0); + + let mut in_off = Cursor::new(0u32); + let mut out_off = Cursor::new(0u32); + + // Write length header then compress. + out[start] = n_values; + self.compress_blocks( + flat, + n_values, + &mut in_off, + &mut out[start + 1..], + &mut out_off, + ); + + let written = 1 + out_off.position() as usize; + out.truncate(start + written); + Ok(()) + } + + fn decode_blocks( + &mut self, + input: &[u32], + expected_len: Option, + out: &mut Vec, + ) -> FastPForResult { + let Some((&block_n_values, rest)) = input.split_first() else { + return Err(FastPForError::NotEnoughData); + }; + if block_n_values % N as u32 != 0 { + return Err(FastPForError::NotEnoughData); + } + if let Some(expected) = expected_len { + if block_n_values != expected { + return Err(FastPForError::DecodedCountMismatch { + actual: block_n_values.as_usize(), + expected: expected.as_usize(), + }); + } + } else { + let max = Self::max_decompressed_len(input.len()); + if block_n_values.as_usize() > max { + return Err(FastPForError::NotEnoughData); + } + } + let n_blocks = block_n_values as usize / N; + if n_blocks == 0 { + return Ok(1); + } + let start = out.len(); + out.resize(start + n_blocks * N, T::zero()); + + let mut in_off = Cursor::new(0u32); + let mut out_off = Cursor::new(0u32); + + self.decode_headless_blocks( + rest, + block_n_values, + &mut in_off, + &mut out[start..], + &mut out_off, + )?; + + let written = out_off.position() as usize; + if written != n_blocks * N { + out.truncate(start + written); + } + // +1 for the header word (block_n_values) that precedes `rest`. + Ok(1 + in_off.position() as usize) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{block_compress, block_decompress, block_roundtrip}; + + #[test] + fn fastpfor_test() { + let mut data = vec![0u32; 256]; + data[126] = u32::MAX; + block_roundtrip::(&data); + } + + #[test] + fn fastpfor_test_128() { + let mut data = vec![0u32; 128]; + data[126] = u32::MAX; + block_roundtrip::(&data); + } + + #[test] + fn test_empty_blocks_ok() { + // Empty input encodes to length header [0] (matches C++ FastPFor) and decodes cleanly. + let enc = block_compress::(&[]).unwrap(); + assert_eq!(enc, [0]); + let dec = block_decompress::(&enc, Some(0)).unwrap(); + assert!(dec.is_empty()); + } + + // Tests ported from C++ + #[test] + fn test_constant_sequence() { + block_roundtrip::(&vec![42u32; 65536]); + } + + #[test] + fn test_alternating_sequence() { + let data: Vec<_> = (0..65536u32).map(|i| u32::from(i % 2 != 0)).collect(); + block_roundtrip::(&data); + } + + #[test] + fn test_large_numbers() { + let data: Vec = (0..65536u32).map(|i| i + (1u32 << 30)).collect(); + block_roundtrip::(&data); + } + + #[test] + fn cursor_api_roundtrip() { + block_roundtrip::(&vec![42u32; 256]); + } + + #[test] + fn headless_compress_unfit_pagesize() { + // 640 values with 128-block codec spans two pages (512 + 128), exercising the loop. + let input: Vec = (0..640u32).collect(); + block_roundtrip::(&input); + } + + #[test] + fn exception_value_vector_resizes() { + // Alternating large/small values trigger exception-buffer resizing across pages. + let input: Vec = (0..1024u32) + .map(|i| if i % 2 == 0 { 1 << 30 } else { 3 }) + .collect(); + block_roundtrip::(&input); + } + + // ── Error / edge tests not covered by `tests/decode_validation.rs` ───── + // + // `AnyLenCodec::decode` treats an empty slice as tail-only and succeeds; an empty + // `decode_blocks` input is still invalid. Headless decode is internal-only. + + #[test] + fn uncompress_zero_input_length_err() { + // Truly empty input (no header word at all) is invalid — C++ would crash reading *in. + block_decompress::(&[], None).unwrap_err(); + } + + #[test] + fn headless_uncompress_zero_inlength_128_ok() { + FastPForBlock128::default() + .decode_headless_blocks( + &[], + 0, + &mut Cursor::new(0u32), + &mut [], + &mut Cursor::new(0u32), + ) + .expect("zero-length decompress must succeed"); + } + + #[test] + fn decode_where_meta_overflow() { + // `decode_headless_blocks` only: no `AnyLenCodec` entry point passes this layout. + let data: Vec = (0..256u32) + .map(|i| if i % 2 == 0 { 1u32 << 30 } else { 3 }) + .collect(); + let compressed = block_compress::(&data).unwrap(); + + let mut padded = vec![0u32]; + padded.extend_from_slice(&compressed); + padded[2] = u32::MAX; + let out_length = padded[1]; + assert!( + FastPForBlock256::default() + .decode_headless_blocks( + &padded, + out_length, + &mut Cursor::new(1u32), + &mut vec![0u32; 320], + &mut Cursor::new(0u32), + ) + .is_err() + ); + } + + #[test] + fn decode_index1_branch_valid() { + let mut data = vec![1u32; 256]; + data[0] = 3; + block_roundtrip::(&data); + } + + /// `decode_blocks` with `expected_len: None` and header=0 returns `Ok` with empty output. + #[test] + fn decode_blocks_header_only_input() { + // Input with just the length header [0]: no blocks to decode. + let input = vec![0u32]; + let out = block_decompress::(&input, None).unwrap(); + assert!(out.is_empty()); + } +} diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs new file mode 100644 index 0000000..2338fb5 --- /dev/null +++ b/src/rust/integer_compression/fastpfor64.rs @@ -0,0 +1,147 @@ +//! 64-bit ([`u64`]) `FastPFOR` block-codec aliases. +//! +//! [`FastPForBlockWide128`]/[`FastPForBlockWide256`] are [`FastPFor`] specialised to `u64` blocks. +//! They implement the block-only [`BlockCodec`](crate::BlockCodec); the width-generic engine and +//! exception bitmap are 64 bits wide instead of 32. +//! The public any-length `u64` codecs [`FastPForWide128`](crate::FastPForWide128) / +//! [`FastPForWide256`](crate::FastPForWide256) pair these with a [`VariableByte`](crate::VariableByte) tail. +//! The block wire format is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. + +use crate::rust::integer_compression::fastpfor::FastPFor; + +/// Type alias for [`FastPFor`] with 128-element `u64` blocks. +pub type FastPForBlockWide128 = FastPFor<128, u64>; + +/// Type alias for [`FastPFor`] with 256-element `u64` blocks. +pub type FastPForBlockWide256 = FastPFor<256, u64>; + +#[cfg(test)] +mod tests { + use crate::codec::BlockCodec64; + use crate::rust::fastpfor_codec::FastPForCodec; + use crate::rust::integer_compression::fastpfor::sealed; + + fn roundtrip(input: &[u64]) + where + [u64; N]: sealed::BlockSize, + { + let mut codec = FastPForCodec::::default(); + let mut encoded = Vec::new(); + codec.encode64(input, &mut encoded).unwrap(); + let mut decoded = Vec::new(); + codec.decode64(&encoded, &mut decoded).unwrap(); + assert_eq!(decoded, input, "roundtrip mismatch (N={N})"); + } + + #[test] + fn empty() { + roundtrip::<128>(&[]); + roundtrip::<256>(&[]); + } + + #[test] + fn single_value() { + roundtrip::<128>(&[42]); + roundtrip::<256>(&[u64::MAX]); + } + + #[test] + fn sub_block_tail_only() { + let data: Vec = (0..10).collect(); + roundtrip::<256>(&data); + } + + #[test] + fn exact_block() { + let data: Vec = (0..128).collect(); + roundtrip::<128>(&data); + } + + #[test] + fn blocks_with_remainder() { + let data: Vec = (0..600).collect(); + roundtrip::<256>(&data); + } + + #[test] + fn large_values_and_exceptions() { + let data: Vec = (0..1024u32) + .map(|i| if i % 7 == 0 { 1u64 << 60 } else { u64::from(i) }) + .collect(); + roundtrip::<128>(&data); + } + + #[test] + fn full_width_values() { + let data: Vec = (0..256u32).map(|i| u64::MAX - u64::from(i)).collect(); + roundtrip::<256>(&data); + } + + #[test] + fn spans_multiple_pages() { + let data: Vec = (0..70_000u64).map(|i| i.wrapping_mul(0x1_0001)).collect(); + roundtrip::<128>(&data); + } + + #[cfg(feature = "cpp")] + mod cpp_parity { + use super::*; + use crate::cpp::{CppFastPFor128, CppFastPFor256}; + + fn cases() -> Vec> { + vec![ + vec![], + vec![42], + vec![u64::MAX], + (0..10).collect(), + (0..128).collect(), + (0..256).collect(), + (0..600).collect(), + (0..1024u32) + .map(|i| if i % 7 == 0 { 1u64 << 60 } else { u64::from(i) }) + .collect(), + (0..256u32).map(|i| u64::MAX - u64::from(i)).collect(), + (0..5000u64).map(|i| i.wrapping_mul(0x1_0001)).collect(), + ] + } + + fn assert_parity( + rust: &mut FastPForCodec, + cpp: &mut impl BlockCodec64, + ) where + [u64; N]: sealed::BlockSize, + { + for data in cases() { + let mut rust_enc = Vec::new(); + rust.encode64(&data, &mut rust_enc).unwrap(); + let mut cpp_enc = Vec::new(); + cpp.encode64(&data, &mut cpp_enc).unwrap(); + assert_eq!(rust_enc, cpp_enc, "encode64 bytes differ for {data:?}"); + + let mut rust_dec = Vec::new(); + rust.decode64(&cpp_enc, &mut rust_dec).unwrap(); + assert_eq!(rust_dec, data, "Rust failed to decode C++ output"); + + let mut cpp_dec = Vec::new(); + cpp.decode64(&rust_enc, &mut cpp_dec).unwrap(); + assert_eq!(cpp_dec, data, "C++ failed to decode Rust output"); + } + } + + #[test] + fn parity_128() { + assert_parity( + &mut FastPForCodec::<128, u64>::default(), + &mut CppFastPFor128::default(), + ); + } + + #[test] + fn parity_256() { + assert_parity( + &mut FastPForCodec::<256, u64>::default(), + &mut CppFastPFor256::default(), + ); + } + } +} diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs new file mode 100644 index 0000000..a74f8d1 --- /dev/null +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -0,0 +1,142 @@ +//! Element-width abstraction shared by the 32- and 64-bit `FastPFOR` codecs. + +use std::array; +use std::fmt::Debug; +use std::ops::{BitOrAssign, Index, IndexMut}; + +use num_traits::PrimInt; + +use crate::helpers::GetWithErr; +use crate::rust::integer_compression::{bit_pack32, bit_pack64, bit_unpack32}; +use crate::{FastPForError, FastPForResult}; + +mod sealed { + pub trait Sealed {} + impl Sealed for u32 {} + impl Sealed for u64 {} +} + +/// Sealed element type of a `FastPFOR` stream: [`u32`] or [`u64`]. +pub trait FastPForInt: PrimInt + 'static + bytemuck::Pod + sealed::Sealed + BitOrAssign { + /// Bit width of the element: 32 or 64. + const WIDTH: u8 = (size_of::() * 8) as u8; + /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. + const BITMAP_WORDS: u32 = Self::WIDTH as u32 / u32::BITS; + + /// Exception values grouped by bit-width bucket: `[Vec; WIDTH + 1]`. + type ExceptionBuffers: Index> + IndexMut + Debug; + /// Per-bit-width frequency counts: `[u32; WIDTH + 1]`. + type Freqs: Index + IndexMut + AsMut<[u32]> + Debug; + /// Write positions into `ExceptionBuffers`: `[usize; WIDTH + 1]`. + type DataPointers: Index + IndexMut + AsMut<[usize]> + Debug; + + /// Fresh, empty exception buffers. + fn new_exception_buffers() -> Self::ExceptionBuffers; + /// Fresh, zeroed frequency counts. + fn new_freqs() -> Self::Freqs; + /// Fresh, zeroed data pointers. + fn new_data_pointers() -> Self::DataPointers; + + /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). + #[inline] + fn significant_bits(self) -> u8 { + Self::WIDTH - self.leading_zeros() as u8 + } + + /// Pack 32 values at `bit` bits each into `out`. + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); + /// Unpack 32 values at `bit` bits each from `src`. + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8); + + /// Write the exception bitmap as [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `out`. + fn write_bitmap(bitmap: Self, out: &mut [u32]); + /// Read the exception bitmap from [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `pos`. + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; +} + +#[allow( + clippy::use_self, + reason = "u32 here is the stream word type, not the Self element type" +)] +impl FastPForInt for u32 { + type ExceptionBuffers = [Vec; u32::BITS as usize + 1]; + type Freqs = [u32; u32::BITS as usize + 1]; + type DataPointers = [usize; u32::BITS as usize + 1]; + + fn new_exception_buffers() -> Self::ExceptionBuffers { + array::from_fn(|_| Vec::new()) + } + fn new_freqs() -> Self::Freqs { + [0; u32::BITS as usize + 1] + } + fn new_data_pointers() -> Self::DataPointers { + [0; u32::BITS as usize + 1] + } + + // `inline(always)`: this is a thin forwarder; without it the wrapper accumulates the whole + // inlined kernel and then exceeds the inline threshold, so `decode_page`/`encode_page` would + // emit a real call per 32-value group instead of inlining the kernel (as the concrete `u32` + // code on `main` does). See the packing-kernel benchmarks. + #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { + bit_pack32::fast_pack(src, inpos, out, outpos, bit); + } + #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { + bit_unpack32::fast_unpack(src, inpos, out, outpos, bit); + } + #[inline] + fn write_bitmap(bitmap: Self, out: &mut [u32]) { + out[0] = bitmap; + } + #[inline] + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + input.get_val(pos) + } +} + +#[allow( + clippy::use_self, + reason = "u32 here is the stream word type, not the Self element type" +)] +impl FastPForInt for u64 { + type ExceptionBuffers = [Vec; u64::BITS as usize + 1]; + type Freqs = [u32; u64::BITS as usize + 1]; + type DataPointers = [usize; u64::BITS as usize + 1]; + + fn new_exception_buffers() -> Self::ExceptionBuffers { + array::from_fn(|_| Vec::new()) + } + fn new_freqs() -> Self::Freqs { + [0; u64::BITS as usize + 1] + } + fn new_data_pointers() -> Self::DataPointers { + [0; u64::BITS as usize + 1] + } + + // `inline(always)`: forward directly to the wide kernel (see the `u32` impl for rationale). + #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { + bit_pack64::pack_wide(src, inpos, out, outpos, bit); + } + #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { + bit_pack64::unpack_wide(src, inpos, out, outpos, bit); + } + #[inline] + fn write_bitmap(bitmap: Self, out: &mut [u32]) { + out[0] = bitmap as u32; + out[1] = (bitmap >> 32) as u32; + } + #[inline] + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + let lo: u32 = input.get_val(pos)?; + let hi_pos = pos.checked_add(1).ok_or(FastPForError::NotEnoughData)?; + let hi: u32 = input.get_val(hi_pos)?; + Ok(u64::from(lo) | (u64::from(hi) << 32)) + } +} diff --git a/src/rust/integer_compression/just_copy.rs b/src/rust/integer_compression/just_copy.rs index 767ad11..64137ef 100644 --- a/src/rust/integer_compression/just_copy.rs +++ b/src/rust/integer_compression/just_copy.rs @@ -17,6 +17,8 @@ impl JustCopy { } impl AnyLenCodec for JustCopy { + type Elem = u32; + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { out.extend_from_slice(input); Ok(()) diff --git a/src/rust/integer_compression/mod.rs b/src/rust/integer_compression/mod.rs index 7aed001..7de1cc6 100644 --- a/src/rust/integer_compression/mod.rs +++ b/src/rust/integer_compression/mod.rs @@ -1,5 +1,9 @@ -pub mod bitpacking; -pub mod bitunpacking; +pub mod bit_pack32; +pub mod bit_pack64; +pub mod bit_unpack32; pub mod fastpfor; +pub mod fastpfor32; +pub mod fastpfor64; +pub mod fastpfor_int; pub mod just_copy; pub mod variable_byte; diff --git a/src/rust/integer_compression/variable_byte.rs b/src/rust/integer_compression/variable_byte.rs index 8c6b2f2..6737c97 100644 --- a/src/rust/integer_compression/variable_byte.rs +++ b/src/rust/integer_compression/variable_byte.rs @@ -1,4 +1,5 @@ use std::io::Cursor; +use std::marker::PhantomData; use bytemuck::{cast_slice, cast_slice_mut}; @@ -7,12 +8,27 @@ use crate::helpers::AsUsize; use crate::rust::cursor::IncrementCursor; use crate::{FastPForError, FastPForResult}; -/// Variable-byte encoding codec for integer compression. -#[derive(Debug, Default)] -pub struct VariableByte; +/// Variable-byte encoding codec, generic over element width `T` ([`u32`] or [`u64`]). +#[derive(Debug)] +pub struct VariableByte(PhantomData); + +// Hand-written (not derived) so no `T: Default` bound leaks onto every user of the codec. +impl Default for VariableByte { + fn default() -> Self { + Self(PhantomData) + } +} + +impl VariableByte { + /// Creates a new instance. + #[must_use] + pub fn new() -> Self { + Self(PhantomData) + } +} // Helper functions with const generics for extracting 7-bit chunks -impl VariableByte { +impl VariableByte { /// Extract 7 bits from position i (with masking) const fn extract_7bits(val: u32) -> u8 { ((val >> (7 * I)) & ((1 << 7) - 1)) as u8 @@ -22,15 +38,6 @@ impl VariableByte { const fn extract_7bits_maskless(val: u32) -> u8 { (val >> (7 * I)) as u8 } -} - -// Implemented for consistency with other codecs -impl VariableByte { - /// Creates a new instance - #[must_use] - pub fn new() -> Self { - Self - } /// Compress `input_length` u32 values from `input[input_offset..]` into /// `output[output_offset..]` as packed variable-byte u8 values (stored in @@ -301,7 +308,9 @@ impl VariableByte { } } -impl AnyLenCodec for VariableByte { +impl AnyLenCodec for VariableByte { + type Elem = u32; + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { let capacity = input.len() * 2 + 4; let start = out.len(); @@ -352,6 +361,73 @@ impl AnyLenCodec for VariableByte { } } +impl AnyLenCodec for VariableByte { + type Elem = u64; + + fn encode(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { + if input.is_empty() { + return Ok(()); + } + let start = out.len(); + let capacity = input.len() * 3 + 4; + out.resize(start + capacity, 0); + let bytes: &mut [u8] = cast_slice_mut(&mut out[start..]); + let mut byte_pos = 0; + for &value in input { + let mut v = value; + while v >= 0x80 { + bytes[byte_pos] = (v as u8) & 0x7F; + byte_pos += 1; + v >>= 7; + } + bytes[byte_pos] = (v as u8) | 0x80; + byte_pos += 1; + } + while byte_pos % 4 != 0 { + bytes[byte_pos] = 0; + byte_pos += 1; + } + out.truncate(start + byte_pos / 4); + Ok(()) + } + + fn decode( + &mut self, + input: &[u32], + out: &mut Vec, + _expected_len: Option, + ) -> FastPForResult<()> { + if input.is_empty() { + return Ok(()); + } + let bytes: &[u8] = cast_slice(input); + let byte_len = bytes.len(); + let mut byte_pos = 0; + while byte_pos < byte_len { + let mut v: u64 = 0; + let mut shift = 0u32; + loop { + if byte_pos >= byte_len { + return Ok(()); + } + let c = bytes[byte_pos]; + byte_pos += 1; + if shift >= 64 { + return Err(FastPForError::NotEnoughData); + } + if c >= 0x80 { + v |= u64::from(c & 0x7F) << shift; + out.push(v); + break; + } + v |= u64::from(c) << shift; + shift += 7; + } + } + Ok(()) + } +} + #[cfg(test)] mod tests { use std::collections::hash_map::RandomState; diff --git a/src/rust/mod.rs b/src/rust/mod.rs index 92a2b05..fe9fbc5 100644 --- a/src/rust/mod.rs +++ b/src/rust/mod.rs @@ -1,17 +1,16 @@ mod composite; mod cursor; +mod fastpfor_codec; mod integer_compression; pub use composite::CompositeCodec; +/// Any-length `FastPFOR` codecs: `FastPFor*` for `u32`, `FastPForWide*` for `u64`. +pub use fastpfor_codec::{FastPFor128, FastPFor256, FastPForWide128, FastPForWide256}; /// Type-safe block codec with block size encoded in the type. -pub use integer_compression::fastpfor::{FastPFor, FastPForBlock128, FastPForBlock256}; +pub use integer_compression::fastpfor::FastPFor; +pub use integer_compression::fastpfor32::{FastPForBlock128, FastPForBlock256}; +pub use integer_compression::fastpfor64::{FastPForBlockWide128, FastPForBlockWide256}; /// Pass-through codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). pub use integer_compression::just_copy::JustCopy; /// Variable-byte codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). pub use integer_compression::variable_byte::VariableByte; - -/// `FastPForBlock256` blocks + `VariableByte` remainder — the most common composite. -pub type FastPFor256 = CompositeCodec; - -/// `FastPForBlock128` blocks + `VariableByte` remainder. -pub type FastPFor128 = CompositeCodec; diff --git a/src/test_utils.rs b/src/test_utils.rs index 70b474d..0c5c704 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -10,10 +10,10 @@ // noise without benefit. #![allow(dead_code, missing_docs, clippy::unwrap_used)] -#[cfg(feature = "cpp")] -use fastpfor::BlockCodec64; #[allow(unused_imports)] -use fastpfor::{AnyLenCodec, BlockCodec, FastPForError, FastPForResult, slice_to_blocks}; +use fastpfor::{ + AnyLenCodec, BlockCodec, BlockCodec64, FastPForError, FastPForResult, slice_to_blocks, +}; #[cfg(feature = "rust")] use fastpfor::{ FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, JustCopy, VariableByte, @@ -25,17 +25,20 @@ pub const RNG_SEED: u64 = 456; // Generic codec helpers // --------------------------------------------------------------------------- -pub fn roundtrip(data: &[u32]) { +pub fn roundtrip>(data: &[u32]) { roundtrip_expected::(data, Some(data.len().try_into().unwrap())); } /// Encode `data` with a caller-owned codec, decode with `expected_len: None`, assert round-trip. -pub fn roundtrip_expected(data: &[u32], expected_len: Option) { +pub fn roundtrip_expected>(data: &[u32], expected_len: Option) { roundtrip_full::(data, expected_len); } /// Encode `data` with a caller-owned codec, decode with `expected_len: None`, assert round-trip. -pub fn roundtrip_full(data: &[u32], expected_len: Option) { +pub fn roundtrip_full, D: AnyLenCodec>( + data: &[u32], + expected_len: Option, +) { let mut encoder = E::default(); let mut compressed = Vec::new(); encoder.encode(data, &mut compressed).unwrap(); @@ -58,19 +61,19 @@ pub fn roundtrip64(data: &[u64]) { assert_eq!(decoded, data); } -pub fn block_roundtrip(data: &[u32]) { +pub fn block_roundtrip>(data: &[u32]) { let compressed = block_compress::(data).unwrap(); let decompressed = block_decompress::(&compressed, Some(data.len() as u32)).unwrap(); assert_eq!(decompressed, data); } -pub fn compress(data: &[u32]) -> FastPForResult> { +pub fn compress>(data: &[u32]) -> FastPForResult> { let mut compressed = Vec::new(); C::default().encode(data, &mut compressed)?; Ok(compressed) } -pub fn decompress( +pub fn decompress>( compressed: &[u32], expected_len: Option, ) -> FastPForResult> { @@ -79,7 +82,7 @@ pub fn decompress( Ok(decompressed) } -pub fn block_compress(data: &[u32]) -> FastPForResult> { +pub fn block_compress>(data: &[u32]) -> FastPForResult> { let (blocks, remainder) = slice_to_blocks::(data); if !remainder.is_empty() { return Err(FastPForError::InputMustBeMultipleOfBlockSize { @@ -92,7 +95,7 @@ pub fn block_compress(data: &[u32]) -> FastPForResult> { Ok(out) } -pub fn block_decompress( +pub fn block_decompress>( compressed: &[u32], expected_len: Option, ) -> FastPForResult> { @@ -142,8 +145,8 @@ pub fn block_roundtrip_all(data: &[u32]) { #[cfg(feature = "rust")] pub fn roundtrip_composite(data: &[u32]) where - B: BlockCodec, - T: AnyLenCodec, + B: BlockCodec, + T: AnyLenCodec, { roundtrip::>(data); } @@ -315,7 +318,7 @@ mod rust_bench { _codec: PhantomData, } - impl CompressFixture { + impl> CompressFixture { fn new(name: &'static str, generator: DataGeneratorFn, block_count: usize) -> Self { let original = generator(block_count * C::size()); Self { @@ -328,7 +331,7 @@ mod rust_bench { } } - impl BlockSizeFixture { + impl> BlockSizeFixture { pub fn new(block_count: usize) -> Self { let original = generate_uniform_data_small_value_distribution(block_count * C::size()); Self { @@ -340,7 +343,7 @@ mod rust_bench { } } - pub fn compress_fixtures( + pub fn compress_fixtures>( block_counts: &[usize], ) -> Vec<(usize, CompressFixture)> { block_counts @@ -353,7 +356,9 @@ mod rust_bench { .collect() } - pub fn ratio_fixtures(block_count: usize) -> Vec> { + pub fn ratio_fixtures>( + block_count: usize, + ) -> Vec> { ALL_PATTERNS .iter() .map(|&(name, generator)| CompressFixture::::new(name, generator, block_count))