fix(lucene): normalize prefix and wildcard queries (alibaba#458)

Author: lszskye

crates/tantivy_ffi/src/reader.rs

@@ -21,8 +21,9 @@
 //!   1 MATCH_ALL — tokenize query, BooleanQuery (Must)
 //!   2 MATCH_ANY — tokenize query, BooleanQuery (Should)
 //!   3 PHRASE    — tokenize query, PhraseQuery
-//!   4 PREFIX    — RegexQuery `<escaped>.*` (no tokenization, mirrors lucene-fts)
-//!   5 WILDCARD  — RegexQuery from glob pattern (`*` → `.*`, `?` → `.`, others escaped)
+//!   4 PREFIX    — original + case-normalized RegexQuery `<escaped>.*` (no tokenization)
+//!   5 WILDCARD  — original + case-normalized RegexQuery from glob pattern
+//!                 (`*` → `.*`, `?` → `.`, others escaped)
 //!
 //! For paimon-java compatibility, row_id is stored as an explicit u64 field
 //! (`fast` for O(1) retrieval). Reader translates tantivy DocAddress → row_id
@@ -37,7 +38,9 @@ use std::path::Path;
 use croaring::{Portable, Treemap};
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::columnar::Column;
-use tantivy::query::{BooleanQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery};
+use tantivy::query::{
+    BooleanQuery, DisjunctionMaxQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery,
+};
 use tantivy::schema::{Field, IndexRecordOption};
 use tantivy::{DocAddress, DocId, Index, IndexReader, ReloadPolicy, Score, SegmentOrdinal,
                SegmentReader, Term};
@@ -46,7 +49,7 @@ use crate::buffer::PaimonTantivyBuffer;
 use crate::callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks};
 use crate::error::{set_last_error, PaimonTantivyStatus};
 use crate::handle::{borrow_handle_mut, free_handle, into_handle};
-use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode};
+use crate::tokenizer::{normalize_case, PaimonJiebaTokenizer, TokenizeMode};
 use crate::writer::{PAIMON_ROW_ID_FIELD_NAME, PAIMON_TEXT_FIELD_NAME, PAIMON_TOKENIZER_NAME};
 
 /// Numeric encoding of `paimon::FullTextSearch::SearchType`. Kept in sync
@@ -229,22 +232,44 @@ impl PaimonTantivyReader {
         if query.is_empty() {
             return Err("prefix query is empty".into());
         }
-        // Mirror lucene-fts: don't tokenize prefix; match indexed term bytes
-        // starting with the given prefix verbatim.
-        let pattern = format!("{}.*", regex_escape(query));
-        RegexQuery::from_pattern(&pattern, self.text_field)
-            .map(|q| Box::new(q) as Box<dyn Query>)
-            .map_err(|e| format!("RegexQuery from prefix {query:?}: {e}"))
+        // Mirror lucene-fts: don't tokenize the prefix. Retain the original form for mixed
+        // ASCII/CJK indexed terms, and add the normalized form for pure ASCII terms.
+        let normalized_query = normalize_case(query);
+        let create_query = |value: &str| -> Result<Box<dyn Query>, String> {
+            let pattern = format!("{}.*", regex_escape(value));
+            RegexQuery::from_pattern(&pattern, self.text_field)
+                .map(|q| Box::new(q) as Box<dyn Query>)
+                .map_err(|e| format!("RegexQuery from prefix {value:?}: {e}"))
+        };
+        let original = create_query(query)?;
+        if normalized_query == query {
+            return Ok(original);
+        }
+        let normalized = create_query(&normalized_query)?;
+        Ok(Box::new(DisjunctionMaxQuery::new(vec![
+            original, normalized,
+        ])))
     }
 
     fn build_wildcard_query(&self, query: &str) -> Result<Box<dyn Query>, String> {
         if query.is_empty() {
             return Err("wildcard query is empty".into());
         }
-        let pattern = wildcard_to_regex(query);
-        RegexQuery::from_pattern(&pattern, self.text_field)
-            .map(|q| Box::new(q) as Box<dyn Query>)
-            .map_err(|e| format!("RegexQuery from wildcard {query:?} (pattern {pattern}): {e}"))
+        let normalized_query = normalize_wildcard_query(query);
+        let create_query = |value: &str| -> Result<Box<dyn Query>, String> {
+            let pattern = wildcard_to_regex(value);
+            RegexQuery::from_pattern(&pattern, self.text_field)
+                .map(|q| Box::new(q) as Box<dyn Query>)
+                .map_err(|e| format!("RegexQuery from wildcard {value:?} (pattern {pattern}): {e}"))
+        };
+        let original = create_query(query)?;
+        if normalized_query == query {
+            return Ok(original);
+        }
+        let normalized = create_query(&normalized_query)?;
+        Ok(Box::new(DisjunctionMaxQuery::new(vec![
+            original, normalized,
+        ])))
     }
 
     fn build_query(&self, search_type: SearchType, query: &str) -> Result<Box<dyn Query>, String> {
@@ -471,6 +496,23 @@ fn regex_escape(input: &str) -> String {
     out
 }
 
+/// Normalize each literal segment independently while preserving wildcard
+/// operators. This matches lucene-fts, where `*` and `?` are not analyzed.
+fn normalize_wildcard_query(input: &str) -> String {
+    let mut out = String::with_capacity(input.len());
+    let mut segment_start = 0;
+    for (index, ch) in input.char_indices() {
+        if ch != '*' && ch != '?' {
+            continue;
+        }
+        out.push_str(&normalize_case(&input[segment_start..index]));
+        out.push(ch);
+        segment_start = index + ch.len_utf8();
+    }
+    out.push_str(&normalize_case(&input[segment_start..]));
+    out
+}
+
 /// Translate a glob-style wildcard ('*' = any, '?' = single char) into a
 /// regex pattern, escaping all other regex metacharacters.
 fn wildcard_to_regex(input: &str) -> String {
@@ -1024,6 +1066,14 @@ mod tests {
         assert_eq!(ids, vec![0u64]);
     }
 
+    #[test]
+    fn prefix_normalizes_ascii_alphanumeric_case() {
+        let bytes = build(&["This is a test document", "another document"]);
+        let r = open(&bytes);
+        let ids = r.search_all(SearchType::Prefix, "THIS").unwrap();
+        assert_eq!(ids, vec![0u64]);
+    }
+
     #[test]
     fn wildcard_with_star() {
         let bytes = build(&["unordered", "ordered", "border"]);
@@ -1032,6 +1082,34 @@ mod tests {
         assert_eq!(ids, vec![0u64, 1, 2]);
     }
 
+    #[test]
+    fn wildcard_normalizes_ascii_alphanumeric_segments() {
+        let bytes = build(&["This is a test document", "another document"]);
+        let r = open(&bytes);
+        assert_eq!(
+            r.search_all(SearchType::Wildcard, "*THIS*").unwrap(),
+            vec![0u64]
+        );
+        assert_eq!(
+            r.search_all(SearchType::Wildcard, "*?HIS*").unwrap(),
+            vec![0u64]
+        );
+    }
+
+    #[test]
+    fn prefix_and_wildcard_preserve_mixed_ascii_cjk_terms() {
+        let bytes = build(&["B超检查", "T恤"]);
+        let r = open(&bytes);
+        assert_eq!(
+            r.search_all(SearchType::Prefix, "B").unwrap(),
+            vec![0u64]
+        );
+        assert_eq!(
+            r.search_all(SearchType::Wildcard, "*T*").unwrap(),
+            vec![1u64]
+        );
+    }
+
     #[test]
     fn empty_query_for_match_returns_query_parse_error() {
         let bytes = build(&["hello"]);
@@ -1048,6 +1126,13 @@ mod tests {
         assert_eq!(wildcard_to_regex("*a*"), ".*a.*");
     }
 
+    #[test]
+    fn wildcard_normalization_preserves_non_alphanumeric_segments() {
+        assert_eq!(normalize_wildcard_query("*?HIS*"), "*?his*");
+        assert_eq!(normalize_wildcard_query("*THIS_IS*"), "*THIS_IS*");
+        assert_eq!(normalize_wildcard_query("*机器*"), "*机器*");
+    }
+
     // ----- limit + pre_filter + scoring (row_id-based) -----
 
     #[test]

crates/tantivy_ffi/src/tokenizer.rs

@@ -120,11 +120,7 @@ impl PaimonJiebaTokenizer {
             let start = piece.as_ptr() as usize - text_start;
             let end = start + piece.len();
             // lowercase only if pure ASCII alphanumeric (match cppjieba Normalize behavior)
-            let token_text = if is_ascii_alnum(piece) {
-                piece.to_ascii_lowercase()
-            } else {
-                piece.to_string()
-            };
+            let token_text = normalize_case(piece);
             out.push((start, end, token_text));
         }
         out
@@ -135,6 +131,14 @@ fn is_ascii_alnum(s: &str) -> bool {
     !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric())
 }
 
+pub(crate) fn normalize_case(s: &str) -> String {
+    if is_ascii_alnum(s) {
+        s.to_ascii_lowercase()
+    } else {
+        s.to_string()
+    }
+}
+
 fn load_jieba(dict_dir: &Path) -> Result<Jieba, String> {
     let main_dict = dict_dir.join("jieba.dict.utf8");
     let mut jieba = if main_dict.exists() {

docs/source/user_guide/global_index.rst

@@ -71,8 +71,15 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie
 - ``MATCH_ALL``: All terms in the query must be present (AND semantics).
 - ``MATCH_ANY``: Any term in the query can match (OR semantics).
 - ``PHRASE``: Matches the exact sequence of words (with proximity).
-- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner).
-- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple").
+- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). The
+  query is not tokenized. The original prefix is retained, and a pure ASCII alphanumeric prefix
+  is also matched using the lowercase case-normalization applied to pure ASCII terms at indexing
+  time. This preserves matches for mixed terms such as ``B超`` while allowing ``THIS`` to match
+  terms indexed as ``this...``.
+- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). The query
+  is not tokenized, and wildcard operators are preserved. Both the original pattern and an
+  alternative with each ASCII alphanumeric fragment lowercased are matched, covering pure ASCII
+  and mixed ASCII/non-ASCII terms.
 
 **Special Configuration:**
 

include/paimon/predicate/full_text_search.h

@@ -77,10 +77,14 @@ struct PAIMON_EXPORT FullTextSearch {
     /// - For PHRASE: matches the exact word sequence (with optional slop). Also be analyzed.
     ///
     /// - For PREFIX: matches terms starting with the given string (e.g., "run" → running, runner).
-    ///   Only the prefix part is considered; analysis will not be applied.
+    ///   The query is not tokenized or filtered for stop words. The original prefix is retained,
+    ///   and a prefix consisting entirely of ASCII letters and digits is also matched using the
+    ///   lowercase case-normalization applied to pure ASCII terms at indexing time.
     ///
     /// - For WILDCARD: supports wildcards * and ? (e.g., "ap*e", "app?e").
-    ///   Not passed through analyzer — matched directly against indexed terms.
+    ///   The query is not tokenized or filtered for stop words. The wildcard operators are
+    ///   preserved. Both the original pattern and an alternative with each ASCII alphanumeric
+    ///   fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms.
     ///
     /// @note Analyzer consistency between indexing and querying is critical for correctness.
     std::string query;

src/paimon/global_index/lucene/jieba_analyzer.cpp

@@ -83,6 +83,21 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb
     }
 }
 
+void JiebaTokenizer::NormalizeCase(std::string* term) {
+    bool is_alphanumeric = true;
+    for (const auto& c : *term) {
+        if (!std::isalnum(static_cast<unsigned char>(c))) {
+            is_alphanumeric = false;
+            break;
+        }
+    }
+    if (is_alphanumeric && !term->empty()) {
+        std::transform(term->begin(), term->end(), term->begin(), [](char ch) {
+            return static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
+        });
+    }
+}
+
 void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words,
                                std::vector<std::string>* input_ptr,
                                std::vector<std::string_view>* output_ptr) {
@@ -98,19 +113,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words
         if (stop_words.find(term) != stop_words.end()) {
             continue;
         }
-        // to lower case
-        bool is_alphanumeric = true;
-        for (const auto& c : term) {
-            if (!std::isalnum(static_cast<unsigned char>(c))) {
-                is_alphanumeric = false;
-                break;
-            }
-        }
-        if (is_alphanumeric && !term.empty()) {
-            std::transform(term.begin(), term.end(), term.begin(), [](char ch) {
-                return static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
-            });
-        }
+        NormalizeCase(&term);
         output.emplace_back(term.data(), term.length());
     }
 }

src/paimon/global_index/lucene/jieba_analyzer.h

@@ -53,6 +53,8 @@ class JiebaTokenizer : public Lucene::Tokenizer {
     static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba,
                             const std::string& str, std::vector<std::string>* terms_ptr);
 
+    static void NormalizeCase(std::string* term);
+
     // In-place converts each string in `input` to lowercase to avoid data copying.
     static void Normalize(const std::unordered_set<std::string>& stop_words,
                           std::vector<std::string>* input, std::vector<std::string_view>* output);

src/paimon/global_index/lucene/jieba_analyzer_test.cpp

@@ -107,6 +107,24 @@ TEST_P(JiebaAnalyzerTest, TestNormalize) {
     ASSERT_EQ(expected, results);
 }
 
+TEST(JiebaTokenizerTest, TestNormalizeCase) {
+    std::string alphanumeric_term = "THIS123";
+    JiebaTokenizer::NormalizeCase(&alphanumeric_term);
+    ASSERT_EQ(alphanumeric_term, "this123");
+
+    std::string mixed_ascii_cjk_term = "B超";
+    JiebaTokenizer::NormalizeCase(&mixed_ascii_cjk_term);
+    ASSERT_EQ(mixed_ascii_cjk_term, "B超");
+
+    std::string term_with_underscore = "THIS_IS";
+    JiebaTokenizer::NormalizeCase(&term_with_underscore);
+    ASSERT_EQ(term_with_underscore, "THIS_IS");
+
+    std::string chinese_term = "机器";
+    JiebaTokenizer::NormalizeCase(&chinese_term);
+    ASSERT_EQ(chinese_term, "机器");
+}
+
 INSTANTIATE_TEST_SUITE_P(ReadBufferSize, JiebaAnalyzerTest,
                          ::testing::ValuesIn(std::vector<int32_t>({2, 5, 10, 100})));
 

src/paimon/global_index/lucene/lucene_global_index_reader.cpp

@@ -16,6 +16,7 @@
 #include "paimon/global_index/lucene/lucene_global_index_reader.h"
 
 #include "arrow/c/bridge.h"
+#include "lucene++/DisjunctionMaxQuery.h"
 #include "lucene++/FileUtils.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/path_util.h"
@@ -116,6 +117,25 @@ std::vector<std::wstring> LuceneGlobalIndexReader::TokenizeQuery(const std::stri
     return wterms;
 }
 
+std::string LuceneGlobalIndexReader::NormalizeWildcardQuery(const std::string& query) {
+    std::string normalized_query;
+    normalized_query.reserve(query.size());
+    size_t term_begin = 0;
+    for (size_t i = 0; i <= query.size(); i++) {
+        if (i != query.size() && query[i] != '*' && query[i] != '?') {
+            continue;
+        }
+        std::string term = query.substr(term_begin, i - term_begin);
+        JiebaTokenizer::NormalizeCase(&term);
+        normalized_query.append(term);
+        if (i != query.size()) {
+            normalized_query.push_back(query[i]);
+        }
+        term_begin = i + 1;
+    }
+    return normalized_query;
+}
+
 Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery(
     const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
     assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL ||
@@ -153,15 +173,43 @@ Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery(
 Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery(
     const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
     assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX);
-    return Lucene::newLucene<Lucene::PrefixQuery>(Lucene::newLucene<Lucene::Term>(
-        wfield_name_, LuceneUtils::StringToWstring(full_text_search->query)));
+    auto create_query = [this](const std::string& query) -> Lucene::QueryPtr {
+        return Lucene::newLucene<Lucene::PrefixQuery>(
+            Lucene::newLucene<Lucene::Term>(wfield_name_, LuceneUtils::StringToWstring(query)));
+    };
+    std::string normalized_query = full_text_search->query;
+    JiebaTokenizer::NormalizeCase(&normalized_query);
+    Lucene::QueryPtr query = create_query(full_text_search->query);
+    if (normalized_query == full_text_search->query) {
+        return query;
+    }
+
+    // Preserve the original query for mixed ASCII/CJK indexed terms such as "B超", whose
+    // complete token is not lowercased by the index analyzer. The normalized alternative still
+    // matches pure ASCII terms. DisjunctionMax avoids double-counting scores if both match.
+    auto disjunction = Lucene::newLucene<Lucene::DisjunctionMaxQuery>(0.0);
+    disjunction->add(query);
+    disjunction->add(create_query(normalized_query));
+    return disjunction;
 }
 
 Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery(
     const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
     assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD);
-    return Lucene::newLucene<Lucene::WildcardQuery>(Lucene::newLucene<Lucene::Term>(
-        wfield_name_, LuceneUtils::StringToWstring(full_text_search->query)));
+    auto create_query = [this](const std::string& query) -> Lucene::QueryPtr {
+        return Lucene::newLucene<Lucene::WildcardQuery>(
+            Lucene::newLucene<Lucene::Term>(wfield_name_, LuceneUtils::StringToWstring(query)));
+    };
+    std::string normalized_query = NormalizeWildcardQuery(full_text_search->query);
+    Lucene::QueryPtr query = create_query(full_text_search->query);
+    if (normalized_query == full_text_search->query) {
+        return query;
+    }
+
+    auto disjunction = Lucene::newLucene<Lucene::DisjunctionMaxQuery>(0.0);
+    disjunction->add(query);
+    disjunction->add(create_query(normalized_query));
+    return disjunction;
 }
 
 Result<std::shared_ptr<GlobalIndexResult>> LuceneGlobalIndexReader::SearchWithLimit(