feat(solver): add hybrid index description (#599)

* add graph

* fix bug for None

* add knowledge unit extra

* fix_prompt

* extract common function into benchmark commponent

* format code

* format code

* format code

* fix benchmark knowledge unit

* fix node

* add common component

* Revert "remove local bge model and dependency scikit-learn"

This reverts commit ebb068f.

* remove unused pkg

* add decompose

* change hybrid default config

* add trace log

* fix some bugs

* with prompt

* support benchmark unit

* support kag thinker model

* support kag thinker model config

* remove default host addr and project id

* add kag model infer script

* add kag model infer script

* modify prompt for kag model

* modify hybrid index docs

* use ttl cache

Author: royzhao

kag/common/conf.py

@@ -20,6 +20,7 @@
 from pathlib import Path
 from typing import Union, Optional, Dict
 
+import knext.common.cache
 from knext.project.client import ProjectClient
 
 logger = logging.getLogger()
@@ -206,13 +207,13 @@ def update_conf(self, configs: dict):
 """
 KAG_QA_TASK_CONFIG stores per-task configuration and should be cleaned up after use.
 """
-KAG_QA_TASK_CONFIG: Dict[str, KAGConfigMgr] = {}
+KAG_QA_TASK_CONFIG  = knext.common.cache.LinkCache(maxsize=100, ttl=300)
 KAG_QA_TASK_CONFIG_LOCK = threading.Lock()
 
 
 class KAGConfigAccessor:
     @staticmethod
-    def get_config(task_with_kb_id=None):
+    def get_config(task_with_kb_id=None) -> KAGConfigMgr:
         """
         Get the configuration information.
 

kag/indexer/kag_index_manager.py

@@ -91,7 +91,7 @@ def description(self) -> str:
     @property
     def schema(self) -> str:
         return """
-AtomicQuery(原子问): EntityType
+AtomicQuery(原子问): IndexType
   properties:
     content(内容): Text
       index: TextAndVector
@@ -200,7 +200,7 @@ def description(self) -> str:
     @property
     def schema(self) -> str:
         return """
-Chunk(文本块): EntityType
+Chunk(文本块): IndexType
      properties:
         content(内容): Text
           index: TextAndVector        
@@ -288,7 +288,7 @@ def description(self) -> str:
     @property
     def schema(self) -> str:
         return """
-Table(表格): EntityType
+Table(表格): IndexType
     properties:
         content(内容): Text
           index: TextAndVector
@@ -370,7 +370,7 @@ def description(self) -> str:
     @property
     def schema(self) -> str:
         return """
-Summary(文本摘要): EntityType
+Summary(文本摘要): IndexType
      properties:
         content(内容): Text
           index: TextAndVector
@@ -449,7 +449,7 @@ def description(self) -> str:
     @property
     def schema(self) -> str:
         return """
-Outline(标题大纲): EntityType
+Outline(标题大纲): IndexType
      properties:
         content(内容): Text
           index: TextAndVector
@@ -525,15 +525,42 @@ def description(self) -> str:
     @property
     def schema(self) -> str:
         return """
-Chunk(文本块): EntityType
+Chunk(文本块): IndexType
      properties:
         content(内容): Text
-          index: TextAndVector        
+          index: TextAndVector  
+
+KnowledgeUnit(知识点): IndexType
+     properties:
+        structedContent(结构化文本): Text
+          index: TextAndVector
+        ontology(本体): Text
+        desc(描述): Text
+            index: TextAndVector
+        relatedQuery(关联问): AtomicQuery
+        extendedKnowledge(关联外扩知识点):Text
+        content(内容): Text
+            index: TextAndVector
+        knowledgeType(知识类型): Text
+
+AtomicQuery(原子问): IndexType
+  properties:
+    title(标题): Text
+      index: TextAndVector
+  relations:
+    relateTo(关联文本块): Chunk
+    similar(相似问题): AtomicQuery
+    relatedTo(相关): KnowledgeUnit      
         """
 
     @property
     def index_cost(self) -> str:
         msg = """
+        索引构建的成本：
+
+        1、抽取模型消耗：7B 4634332 tokens
+        2、耗时：1425 秒
+        3、文件字数：10万字
         """
         return msg
 
@@ -552,7 +579,7 @@ def applicable_scenarios(self) -> str:
 
     @property
     def retrieval_method(self) -> str:
-        return "通过构建chunk 与 图谱的关联，实现chunk 的检索，一般用于检索与图谱相关的chunk"
+        return "通过构建chunk 与 图谱的关联，实现图谱、chunk 的检索，一般用于检索与图谱相关的chunk"
 
     @classmethod
     def build_extractor_config(
@@ -616,7 +643,7 @@ def build_retriever_config(
                 "kag_qa_task_config_key": kb_task_project_id,
             },
             {
-                "type": "kg_fr_open_spg",
+                "type": "kg_fr_knowledge_unit",
                 "top_k": 20,
                 "path_select": {
                     "type": "fuzzy_one_hop_select",