fix: add normalize_for_embedding function to clean text input

--bug=1065973@tapd-62980211 --user=刘瑞斌 【知识库】关联问题带特殊符号时，命中测试结果显示错误 https://www.tapd.cn/62980211/s/1841893

Author: liuruibin

apps/knowledge/vector/base_vector.py

@@ -6,6 +6,7 @@
     @date：2023/10/18 19:16
     @desc:
 """
+import re
 import threading
 from abc import ABC, abstractmethod
 from functools import reduce
@@ -33,6 +34,26 @@ def chunk_data_list(data_list: List[Dict]):
     return reduce(lambda x, y: [*x, *y], result, [])
 
 
+# 预编译正则，性能更好
+RE_EMOJI = re.compile(
+    r"[\U0001F300-\U0001FAFF]"  # Emoji
+    r"|[\u2600-\u27BF]"  # Dingbats / Symbols（⚓ 在这）
+    r"|[\uFE0E\uFE0F]",  # Variation Selectors
+    flags=re.UNICODE
+)
+
+RE_WHITESPACE = re.compile(r"\s+")
+
+
+def normalize_for_embedding(text: str) -> str:
+    if not text:
+        return ""
+
+    text = RE_EMOJI.sub("", text)
+    text = RE_WHITESPACE.sub(" ", text)
+    return text.strip()
+
+
 class BaseVectorStore(ABC):
     vector_exists = False
 
@@ -121,6 +142,7 @@ def search(self, query_text, knowledge_id_list: list[str], exclude_document_id_l
                embedding: Embeddings):
         if knowledge_id_list is None or len(knowledge_id_list) == 0:
             return []
+        query_text = normalize_for_embedding(query_text)
         embedding_query = embedding.embed_query(query_text)
         result = self.query(embedding_query, knowledge_id_list, exclude_document_id_list, exclude_paragraph_list,
                             is_active, 1, 3, 0.65)

apps/knowledge/vector/pg_vector.py

@@ -21,7 +21,7 @@
 from common.utils.common import get_file_content
 from common.utils.ts_vecto_util import to_ts_vector, to_query
 from knowledge.models import Embedding, SearchMode, SourceType
-from knowledge.vector.base_vector import BaseVectorStore
+from knowledge.vector.base_vector import BaseVectorStore, normalize_for_embedding
 from maxkb.conf import PROJECT_DIR
 
 
@@ -46,6 +46,7 @@ def _save(self, text, source_type: SourceType, knowledge_id: str, document_id: s
               source_id: str,
               is_active: bool,
               embedding: Embeddings):
+        text = normalize_for_embedding(text)
         text_embedding = [float(x) for x in embedding.embed_query(text)]
         embedding = Embedding(
             id=uuid.uuid7(),
@@ -62,7 +63,7 @@ def _save(self, text, source_type: SourceType, knowledge_id: str, document_id: s
         return True
 
     def _batch_save(self, text_list: List[Dict], embedding: Embeddings, is_the_task_interrupted):
-        texts = [row.get('text') for row in text_list]
+        texts = [normalize_for_embedding(row.get('text')) for row in text_list]
         embeddings = embedding.embed_documents(texts)
         embedding_list = [
             Embedding(
@@ -87,6 +88,7 @@ def hit_test(self, query_text, knowledge_id_list: list[str], exclude_document_id
         if knowledge_id_list is None or len(knowledge_id_list) == 0:
             return []
         exclude_dict = {}
+        query_text = normalize_for_embedding(query_text)
         embedding_query = embedding.embed_query(query_text)
         query_set = QuerySet(Embedding).filter(knowledge_id__in=knowledge_id_list, is_active=True)
         if exclude_document_id_list is not None and len(exclude_document_id_list) > 0: