Skip to content

Commit c2cdf99

Browse files
committed
fix 0436
1 parent 950de26 commit c2cdf99

3 files changed

Lines changed: 9 additions & 7 deletions

File tree

src/paimon/global_index/lucene/jieba_analyzer.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,15 @@ void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words
9898
// to lower case
9999
bool is_alphanumeric = true;
100100
for (const auto& c : term) {
101-
if (!std::isalnum(c)) {
101+
if (!std::isalnum(static_cast<unsigned char>(c))) {
102102
is_alphanumeric = false;
103103
break;
104104
}
105105
}
106106
if (is_alphanumeric && !term.empty()) {
107-
std::transform(term.begin(), term.end(), term.begin(), ::tolower);
107+
std::transform(term.begin(), term.end(), term.begin(), [](char ch) {
108+
return static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
109+
});
108110
}
109111
output.emplace_back(term.data(), term.length());
110112
}

src/paimon/global_index/lucene/lucene_api_test.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,14 +132,14 @@ class LuceneInterfaceTest : public ::testing::Test {
132132
context->writer->addDocument(context->doc);
133133
}
134134

135-
struct ReadConext {
135+
struct ReadContext {
136136
Lucene::IndexReaderPtr reader;
137137
Lucene::IndexSearcherPtr searcher;
138138
Lucene::QueryParserPtr parser;
139139
};
140140

141-
ReadConext CreateReadContext(const Lucene::DirectoryPtr& lucene_dir,
142-
const Lucene::AnalyzerPtr& analyzer) const {
141+
ReadContext CreateReadContext(const Lucene::DirectoryPtr& lucene_dir,
142+
const Lucene::AnalyzerPtr& analyzer) const {
143143
auto lucene_analyzer = analyzer ? analyzer
144144
: Lucene::newLucene<Lucene::StandardAnalyzer>(
145145
Lucene::LuceneVersion::LUCENE_CURRENT);
@@ -155,7 +155,7 @@ class LuceneInterfaceTest : public ::testing::Test {
155155
const std::optional<std::vector<int32_t>> selected_id,
156156
const std::vector<int32_t>& expected_doc_id_vec,
157157
const std::vector<std::wstring>& expected_doc_id_content_vec,
158-
ReadConext* context) const {
158+
ReadContext* context) const {
159159
Lucene::QueryPtr query = context->parser->parse(query_str);
160160
Lucene::TopDocsPtr results;
161161
if (selected_id) {

src/paimon/global_index/lucene/lucene_global_index_writer.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
#include "arrow/type.h"
1919
#include "cppjieba/Jieba.hpp"
2020
#include "lucene++/LuceneHeaders.h"
21-
#include "paimon//global_index/io/global_index_file_writer.h"
2221
#include "paimon/global_index/global_index_writer.h"
22+
#include "paimon/global_index/io/global_index_file_writer.h"
2323
#include "paimon/global_index/lucene/lucene_defs.h"
2424

2525
namespace paimon::lucene {

0 commit comments

Comments
 (0)