fixup

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

_typos.toml

@@ -8,7 +8,7 @@ extend-ignore-re = [
 ]
 
 [files]
-extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**", "vortex-sqllogictest/slt/**"]
+extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**", "vortex-sqllogictest/slt/**", "encodings/fsst/src/dfa/tests.rs"]
 
 [type.py]
 extend-ignore-identifiers-re = [

encodings/fsst/src/compute/like.rs

@@ -33,15 +33,25 @@ impl LikeKernel for FSST {
             return Ok(None);
         }
 
-        let Some(pattern_str) = pattern_scalar.as_utf8().value() else {
+        let pattern_bytes: &[u8] = if let Some(s) = pattern_scalar.as_utf8_opt() {
+            let Some(v) = s.value() else {
+                return Ok(None);
+            };
+            v.as_ref()
+        } else if let Some(b) = pattern_scalar.as_binary_opt() {
+            let Some(v) = b.value() else {
+                return Ok(None);
+            };
+            v
+        } else {
             return Ok(None);
         };
 
         let symbols = array.symbols();
         let symbol_lengths = array.symbol_lengths();
 
         let Some(matcher) =
-            FsstMatcher::try_new(symbols.as_slice(), symbol_lengths.as_slice(), pattern_str)?
+            FsstMatcher::try_new(symbols.as_slice(), symbol_lengths.as_slice(), pattern_bytes)?
         else {
             return Ok(None);
         };
@@ -73,9 +83,6 @@ impl LikeKernel for FSST {
 mod tests {
     use std::sync::LazyLock;
 
-    use rand::Rng;
-    use rand::SeedableRng;
-    use rand::rngs::StdRng;
     use vortex_array::Canonical;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;

encodings/fsst/src/dfa/flat_contains.rs

@@ -12,7 +12,7 @@
 //!
 //! ### Step 1: KMP (Knuth–Morris–Pratt) byte-level transition table
 //!
-//! See: https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm
+//! See: <https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm>
 //!
 //! Build a `(state × byte) → state` table using the KMP failure function.
 //! States 0..2 track match progress, state 3 is accept (sticky).

encodings/fsst/src/dfa/mod.rs

@@ -163,23 +163,21 @@ impl FsstMatcher {
     pub(crate) fn try_new(
         symbols: &[Symbol],
         symbol_lengths: &[u8],
-        pattern: &str,
+        pattern: &[u8],
     ) -> VortexResult<Option<Self>> {
         let Some(like_kind) = LikeKind::parse(pattern) else {
             return Ok(None);
         };
 
         let inner = match like_kind {
-            LikeKind::Prefix("") | LikeKind::Contains("") => MatcherInner::MatchAll,
+            LikeKind::Prefix(b"") | LikeKind::Contains(b"") => MatcherInner::MatchAll,
             LikeKind::Prefix(prefix) => {
-                let prefix = prefix.as_bytes();
                 if prefix.len() > FlatPrefixDfa::MAX_PREFIX_LEN {
                     return Ok(None);
                 }
                 MatcherInner::Prefix(FlatPrefixDfa::new(symbols, symbol_lengths, prefix)?)
             }
             LikeKind::Contains(needle) => {
-                let needle = needle.as_bytes();
                 if needle.len() > FlatContainsDfa::MAX_NEEDLE_LEN {
                     return Ok(None);
                 }
@@ -203,23 +201,24 @@ impl FsstMatcher {
 /// The subset of LIKE patterns we can handle without decompression.
 enum LikeKind<'a> {
     /// `prefix%`
-    Prefix(&'a str),
+    Prefix(&'a [u8]),
     /// `%needle%`
-    Contains(&'a str),
+    Contains(&'a [u8]),
 }
 
 impl<'a> LikeKind<'a> {
-    fn parse(pattern: &'a str) -> Option<Self> {
+    fn parse(pattern: &'a [u8]) -> Option<Self> {
         // `prefix%` (including just `%` where prefix is empty)
-        if let Some(prefix) = pattern.strip_suffix('%')
-            && !prefix.contains(['%', '_'])
+        if let Some(prefix) = pattern.strip_suffix(&[b'%'])
+            && !prefix.contains(&b'%')
+            && !prefix.contains(&b'_')
         {
             return Some(LikeKind::Prefix(prefix));
         }
 
         // `%needle%`
-        let inner = pattern.strip_prefix('%')?.strip_suffix('%')?;
-        if !inner.contains(['%', '_']) {
+        let inner = pattern.strip_prefix(&[b'%'])?.strip_suffix(&[b'%'])?;
+        if !inner.contains(&b'%') && !inner.contains(&b'_') {
             return Some(LikeKind::Contains(inner));
         }
 

encodings/fsst/src/dfa/tests.rs

@@ -6,12 +6,14 @@ use std::sync::LazyLock;
 use fsst::ESCAPE_CODE;
 use fsst::Symbol;
 use rstest::rstest;
+use vortex_array::ArrayRef;
 use vortex_array::Canonical;
 use vortex_array::IntoArray;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::BoolArray;
 use vortex_array::arrays::ConstantArray;
 use vortex_array::arrays::VarBinArray;
+use vortex_array::arrays::scalar_fn::ScalarFnArrayExt;
 use vortex_array::assert_arrays_eq;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::Nullability;
@@ -51,17 +53,17 @@ fn escaped(bytes: &[u8]) -> Vec<u8> {
 #[test]
 fn test_like_kind_parse() {
     assert!(matches!(
-        LikeKind::parse("http%"),
-        Some(LikeKind::Prefix("http"))
+        LikeKind::parse(b"http%"),
+        Some(LikeKind::Prefix(b"http"))
     ));
     assert!(matches!(
-        LikeKind::parse("%needle%"),
-        Some(LikeKind::Contains("needle"))
+        LikeKind::parse(b"%needle%"),
+        Some(LikeKind::Contains(b"needle"))
     ));
-    assert!(matches!(LikeKind::parse("%"), Some(LikeKind::Prefix(""))));
+    assert!(matches!(LikeKind::parse(b"%"), Some(LikeKind::Prefix(b""))));
     // Suffix and underscore patterns are not supported.
-    assert!(LikeKind::parse("%suffix").is_none());
-    assert!(LikeKind::parse("a_c").is_none());
+    assert!(LikeKind::parse(b"%suffix").is_none());
+    assert!(LikeKind::parse(b"a_c").is_none());
 }
 
 /// No symbols — all bytes escaped. Simplest case to see the two tables.
@@ -144,7 +146,7 @@ fn test_prefix_dfa_longer() -> VortexResult<()> {
 
 #[test]
 fn test_prefix_pushdown_len_13_with_escapes() {
-    let matcher = FsstMatcher::try_new(&[], &[], "abcdefghijklm%")
+    let matcher = FsstMatcher::try_new(&[], &[], b"abcdefghijklm%")
         .unwrap()
         .unwrap();
 
@@ -156,7 +158,7 @@ fn test_prefix_pushdown_len_13_with_escapes() {
 fn test_prefix_pushdown_len_14_now_handled() {
     // 14-byte prefix is now handled by FlatPrefixDfa (was rejected by shift-packed).
     assert!(
-        FsstMatcher::try_new(&[], &[], "abcdefghijklmn%")
+        FsstMatcher::try_new(&[], &[], b"abcdefghijklmn%")
             .unwrap()
             .is_some()
     );
@@ -166,7 +168,7 @@ fn test_prefix_pushdown_len_14_now_handled() {
 fn test_prefix_pushdown_long_prefix() -> VortexResult<()> {
     let prefix = "a".repeat(FlatPrefixDfa::MAX_PREFIX_LEN);
     let pattern = format!("{prefix}%");
-    let matcher = FsstMatcher::try_new(&[], &[], &pattern)?.unwrap();
+    let matcher = FsstMatcher::try_new(&[], &[], pattern.as_bytes())?.unwrap();
 
     assert!(matcher.matches(&escaped(prefix.as_bytes())));
 
@@ -182,14 +184,20 @@ fn test_prefix_pushdown_rejects_len_254() {
     debug_assert_eq!(FlatPrefixDfa::MAX_PREFIX_LEN, 253);
     let prefix = "a".repeat(254);
     let pattern = format!("{prefix}%");
-    assert!(FsstMatcher::try_new(&[], &[], &pattern).unwrap().is_none());
+    assert!(
+        FsstMatcher::try_new(&[], &[], pattern.as_bytes())
+            .unwrap()
+            .is_none()
+    );
 }
 
 #[test]
 fn test_contains_pushdown_len_254_with_escapes() {
     let needle = "a".repeat(FlatContainsDfa::MAX_NEEDLE_LEN);
     let pattern = format!("%{needle}%");
-    let matcher = FsstMatcher::try_new(&[], &[], &pattern).unwrap().unwrap();
+    let matcher = FsstMatcher::try_new(&[], &[], pattern.as_bytes())
+        .unwrap()
+        .unwrap();
 
     assert!(matcher.matches(&escaped(needle.as_bytes())));
 
@@ -202,14 +210,18 @@ fn test_contains_pushdown_len_254_with_escapes() {
 fn test_contains_pushdown_rejects_len_255() {
     let needle = "a".repeat(FlatContainsDfa::MAX_NEEDLE_LEN + 1);
     let pattern = format!("%{needle}%");
-    assert!(FsstMatcher::try_new(&[], &[], &pattern).unwrap().is_none());
+    assert!(
+        FsstMatcher::try_new(&[], &[], pattern.as_bytes())
+            .unwrap()
+            .is_none()
+    );
 }
 
 // ---------------------------------------------------------------------------
 // End-to-end edge cases: FSST compress → LIKE → compare booleans
 // ---------------------------------------------------------------------------
 
-fn make_fsst(strings: &[Option<&str>]) -> FSSTArray {
+fn make_fsst_str(strings: &[Option<&str>]) -> FSSTArray {
     let varbin = VarBinArray::from_iter(
         strings.iter().copied(),
         DType::Utf8(Nullability::NonNullable),
@@ -218,13 +230,9 @@ fn make_fsst(strings: &[Option<&str>]) -> FSSTArray {
     fsst_compress(varbin, &compressor)
 }
 
-fn run_like(array: FSSTArray, pattern: &str) -> VortexResult<BoolArray> {
-    use vortex_array::ArrayRef;
-    use vortex_array::arrays::scalar_fn::ScalarFnArrayExt;
-
+fn run_like(array: FSSTArray, pattern_arr: ArrayRef) -> VortexResult<BoolArray> {
     let len = array.len();
     let arr: ArrayRef = array.into_array();
-    let pattern_arr = ConstantArray::new(pattern, len).into_array();
     let result = Like
         .try_new_array(len, LikeOptions::default(), [arr, pattern_arr])?
         .into_array()
@@ -267,14 +275,42 @@ fn run_like(array: FSSTArray, pattern: &str) -> VortexResult<BoolArray> {
 // Prefix that shares chars with rest of string
 #[case(&["abab", "abba", "abcd"], "ab%", &[true, true, true])]
 #[case(&["abab", "abba", "abcd", "ba"], "ab%", &[true, true, true, false])]
+// The string "aabaabaabaab" requires multi-level KMP fallback at the 'a' after "aabaabaab"
+#[case(&["aabaabaabaab", "aabaabaax", "xaabaabaab"], "%aabaabaab%", &[true, false, true])]
+#[case(&["café latte", "naïve approach", "café noir"], "café%", &[true, false, true])]
+#[case(&["日本語テスト", "日本語データ", "英語テスト"], "%日本語%", &[true, true, false])]
+// 10-byte needle, contains: match at start, middle, end, exact, and near-miss
+#[case(
+    &["abcdefghijxxx", "xxxabcdefghij", "xxabcdefghijxx", "abcdefghij", "abcdefghxx"],
+    "%abcdefghij%",
+    &[true, true, true, true, false]
+)]
+// 10-byte prefix: same needle but anchored at the start of the string
+#[case(
+    &["abcdefghijxxx", "abcdefghij", "xabcdefghij", "abcdefghxx"],
+    "abcdefghij%",
+    &[true, true, false, false]
+)]
+// 9-byte needle with KMP-relevant overlap ("abcabcabc"):
+// failure table = [0,0,0,1,2,3,4,5,6], so a partial match of "abcabcab"
+// followed by a mismatch must fall back to state 5 ("abcab"), not restart.
+// This exercises multi-level KMP backtracking across symbol boundaries.
+#[case(
+    &["xxabcabcabcxx", "abcabcabc", "abcabcabx", "abcabcxx"],
+    "%abcabcabc%",
+    &[true, true, false, false]
+)]
 fn test_like_edge_cases(
     #[case] strings: &[&str],
     #[case] pattern: &str,
     #[case] expected: &[bool],
 ) -> VortexResult<()> {
     let opts: Vec<Option<&str>> = strings.iter().map(|s| Some(*s)).collect();
-    let fsst = make_fsst(&opts);
-    let result = run_like(fsst, pattern)?;
+    let fsst_arr = make_fsst_str(&opts);
+    let result = run_like(
+        fsst_arr,
+        ConstantArray::new(pattern, opts.len()).into_array(),
+    )?;
     let expected_arr = BoolArray::from_iter(expected.iter().copied());
     assert_arrays_eq!(&result, &expected_arr);
     Ok(())