Skip to content

Commit 666ad9c

Browse files
author
Claude Agent
committed
Merge Fat Teddy (Task B) multi-pattern OR
Brings in `MultiNeedleMatcher` + `dfa/fat_teddy.rs` (8-bucket Fat Teddy with greedy bucket-packing, AVX2 + scalar, per-bucket `FoldedContainsDfa::matches` verifier) plus 12 new unit/property tests and 6 multi-needle OR benches. Conflicts resolved: - `mod.rs`: kept HEAD's empirically-tuned ShiftOr gate (no escape-only eligible AND no first-byte present in any symbol AND no SSA), plus Task A's `first_byte_present_in_any_symbol` helper; appended Fat Teddy's `MultiNeedleMatcher` section unchanged. - `tests.rs`: HEAD's test bodies covering the richer ShiftOr routing predicates; appended Fat Teddy's `MultiNeedleMatcher` test section (12 new tests). - `benches/fsst_like.rs`: appended Fat Teddy's six `fsst_contains_or_*` benches (3-, 8-, 16-needle Fat Teddy vs N-pass baselines on the ClickBench URL corpus). Deferred TODOs preserved from the subagent's commit: - Cross-bucket FDR for ESCAPE_CODE-anchored needles (falls back to N-pass). - AVX-512 and NEON variants of `fat_teddy_pass_*`. - Planner integration (Task C, separate merge). 196 tests pass with `_test-harness`. `cargo +nightly fmt --all` clean.
2 parents eb2b2e6 + 0e8889c commit 666ad9c

5 files changed

Lines changed: 1424 additions & 21 deletions

File tree

encodings/fsst/benches/fsst_like.rs

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,3 +224,125 @@ fn fsst_contains_short_qq_cb(bencher: Bencher) {
224224
fn fsst_contains_short_xyzz_rare(bencher: Bencher) {
225225
bench_like(bencher, &FSST_RARE_MATCH, "%xyzz%");
226226
}
227+
228+
// ---------------------------------------------------------------------------
229+
// Fat Teddy / multi-needle OR benches
230+
//
231+
// Each `fsst_contains_or_<n>_<dataset>` bench runs `MultiNeedleMatcher`
232+
// (Fat Teddy single pass) on a small needle list, while the
233+
// `fsst_contains_or_<n>_<dataset>_npass` baseline runs the same needles
234+
// as N separate single-pattern `FsstMatcher` scans and bitwise-ORs the
235+
// results. The Fat Teddy variant should be ≥ 1.5× faster than the
236+
// N-pass baseline for n ≥ 4.
237+
// ---------------------------------------------------------------------------
238+
239+
/// Run `MultiNeedleMatcher::scan_or_to_bitbuf` on `fsst` for the given
240+
/// needle list, returning the OR bit-buffer.
241+
fn fat_teddy_or(fsst: &FSSTArray, patterns: &[&str]) -> vortex_buffer::BitBuffer {
242+
#[expect(deprecated)]
243+
use vortex_array::ToCanonical;
244+
use vortex_array::arrays::varbin::VarBinArrayExt;
245+
use vortex_array::match_each_integer_ptype;
246+
use vortex_fsst::FSSTArrayExt;
247+
use vortex_fsst::dfa::MultiNeedleMatcher;
248+
249+
let symbols = fsst.symbols();
250+
let symbol_lengths = fsst.symbol_lengths();
251+
let codes = fsst.codes();
252+
#[expect(deprecated)]
253+
let offsets = codes.offsets().to_primitive();
254+
let all_bytes = codes.bytes();
255+
let all_bytes = all_bytes.as_slice();
256+
let n = codes.len();
257+
let pattern_bytes: Vec<&[u8]> = patterns.iter().map(|s| s.as_bytes()).collect();
258+
let matcher = MultiNeedleMatcher::try_new_multi(
259+
symbols.as_slice(),
260+
symbol_lengths.as_slice(),
261+
&pattern_bytes,
262+
false,
263+
)
264+
.unwrap()
265+
.unwrap();
266+
match_each_integer_ptype!(offsets.ptype(), |T| {
267+
let off = offsets.as_slice::<T>();
268+
matcher.scan_or_to_bitbuf(n, off, all_bytes, false)
269+
})
270+
}
271+
272+
/// Run N separate single-pattern `FsstMatcher` scans and OR-merge them.
273+
fn npass_or(fsst: &FSSTArray, patterns: &[&str]) -> vortex_buffer::BitBuffer {
274+
#[expect(deprecated)]
275+
use vortex_array::ToCanonical;
276+
use vortex_array::arrays::varbin::VarBinArrayExt;
277+
use vortex_array::match_each_integer_ptype;
278+
use vortex_fsst::FSSTArrayExt;
279+
use vortex_fsst::dfa::FsstMatcher;
280+
281+
let symbols = fsst.symbols();
282+
let symbol_lengths = fsst.symbol_lengths();
283+
let codes = fsst.codes();
284+
#[expect(deprecated)]
285+
let offsets = codes.offsets().to_primitive();
286+
let all_bytes = codes.bytes();
287+
let all_bytes = all_bytes.as_slice();
288+
let n = codes.len();
289+
let mut acc: Option<vortex_buffer::BitBuffer> = None;
290+
for p in patterns {
291+
let m = FsstMatcher::try_new_with(
292+
symbols.as_slice(),
293+
symbol_lengths.as_slice(),
294+
p.as_bytes(),
295+
false,
296+
)
297+
.unwrap()
298+
.unwrap();
299+
let r = match_each_integer_ptype!(offsets.ptype(), |T| {
300+
let off = offsets.as_slice::<T>();
301+
m.scan_to_bitbuf(n, off, all_bytes, false)
302+
});
303+
acc = Some(match acc {
304+
Some(prev) => &prev | &r,
305+
None => r,
306+
});
307+
}
308+
acc.unwrap()
309+
}
310+
311+
static NEEDLES_OR_3_URLS: &[&str] = &["%google%", "%yandex%", "%bing%"];
312+
static NEEDLES_OR_8_URLS: &[&str] = &[
313+
"%google%", "%yandex%", "%bing%", "%duck%", "%wiki%", "%news%", "%blog%", "%shop%",
314+
];
315+
static NEEDLES_OR_16_URLS: &[&str] = &[
316+
"%google%", "%yandex%", "%bing%", "%duck%", "%wiki%", "%news%", "%blog%", "%shop%", "%com%",
317+
"%org%", "%net%", "%info%", "%http%", "%https%", "%www%", "%api%",
318+
];
319+
320+
#[divan::bench]
321+
fn fsst_contains_or_3_urls(bencher: Bencher) {
322+
bencher.bench(|| fat_teddy_or(&FSST_CB_URLS, NEEDLES_OR_3_URLS));
323+
}
324+
325+
#[divan::bench]
326+
fn fsst_contains_or_3_urls_npass(bencher: Bencher) {
327+
bencher.bench(|| npass_or(&FSST_CB_URLS, NEEDLES_OR_3_URLS));
328+
}
329+
330+
#[divan::bench]
331+
fn fsst_contains_or_8_urls(bencher: Bencher) {
332+
bencher.bench(|| fat_teddy_or(&FSST_CB_URLS, NEEDLES_OR_8_URLS));
333+
}
334+
335+
#[divan::bench]
336+
fn fsst_contains_or_8_urls_npass(bencher: Bencher) {
337+
bencher.bench(|| npass_or(&FSST_CB_URLS, NEEDLES_OR_8_URLS));
338+
}
339+
340+
#[divan::bench]
341+
fn fsst_contains_or_16_urls(bencher: Bencher) {
342+
bencher.bench(|| fat_teddy_or(&FSST_CB_URLS, NEEDLES_OR_16_URLS));
343+
}
344+
345+
#[divan::bench]
346+
fn fsst_contains_or_16_urls_npass(bencher: Bencher) {
347+
bencher.bench(|| npass_or(&FSST_CB_URLS, NEEDLES_OR_16_URLS));
348+
}

0 commit comments

Comments
 (0)