feat: Knowledge export and import

Author: zhanweizhang7

apps/knowledge/api/knowledge.py

@@ -6,7 +6,7 @@
 from knowledge.serializers.common import BatchSerializer, BatchMoveSerializer
 from knowledge.serializers.common import GenerateRelatedSerializer
 from knowledge.serializers.knowledge import KnowledgeBaseCreateRequest, KnowledgeModelSerializer, KnowledgeEditRequest, \
-    KnowledgeWebCreateRequest, HitTestSerializer
+    KnowledgeWebCreateRequest, HitTestSerializer, KnowledgeImportRequest
 
 
 class KnowledgeCreateResponse(ResultSerializer):
@@ -306,3 +306,26 @@ def get_request():
     @staticmethod
     def get_move_request():
         return BatchMoveSerializer
+
+
+class KnowledgeImportAPI(APIMixin):
+    @staticmethod
+    def get_parameters():
+        return [
+            OpenApiParameter(
+                name="workspace_id",
+                description="工作空间id",
+                type=OpenApiTypes.STR,
+                location='path',
+                required=True,
+            ),
+        ]
+
+    @staticmethod
+    def get_request():
+        return KnowledgeImportRequest
+
+    @staticmethod
+    def get_response():
+        return DefaultResultSerializer
+

apps/knowledge/serializers/knowledge.py

@@ -17,6 +17,7 @@
     path('workspace/<str:workspace_id>/knowledge/tags', views.KnowledgeView.Tags.as_view()),
     path('workspace/<str:workspace_id>/knowledge/batch_delete', views.KnowledgeView.BatchDelete.as_view()),
     path('workspace/<str:workspace_id>/knowledge/batch_move', views.KnowledgeView.BatchMove.as_view()),
+    path('workspace/<str:workspace_id>/knowledge/import_knowledge', views.KnowledgeView.ImportKnowledge.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>', views.KnowledgeView.Operate.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/sync', views.KnowledgeView.SyncWeb.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/workflow', views.KnowledgeWorkflowView.Operate.as_view()),
@@ -27,6 +28,7 @@
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/hit_test', views.KnowledgeView.HitTest.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export', views.KnowledgeView.Export.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export_zip', views.KnowledgeView.ExportZip.as_view()),
+    path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export_knowledge', views.KnowledgeView.ExportKnowledge.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/transform_workflow', views.KnowledgeView.TransformWorkflow.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/tags', views.KnowledgeTagView.as_view()),
     path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/tags/batch_delete', views.KnowledgeTagView.BatchDelete.as_view()),

apps/knowledge/urls.py

@@ -1,16 +1,17 @@
 from django.utils.translation import gettext_lazy as _
 from drf_spectacular.utils import extend_schema
+from rest_framework.parsers import MultiPartParser
 from rest_framework.request import Request
 from rest_framework.views import APIView
 
 from common.auth import TokenAuth
-from common.auth.authentication import has_permissions, check_batch_permissions
+from common.auth.authentication import has_permissions, check_batch_permissions, get_is_permissions
 from common.constants.permission_constants import PermissionConstants, RoleConstants, ViewPermission, CompareConstants
 from common.log.log import log
 from common import result
 from knowledge.api.knowledge import KnowledgeBaseCreateAPI, KnowledgeWebCreateAPI, KnowledgeTreeReadAPI, \
     KnowledgeEditAPI, KnowledgeReadAPI, KnowledgePageAPI, SyncWebAPI, GenerateRelatedAPI, HitTestAPI, EmbeddingAPI, \
-    GetModelAPI, KnowledgeExportAPI, KnowledgeBatchOperateAPI
+    GetModelAPI, KnowledgeExportAPI, KnowledgeBatchOperateAPI, KnowledgeImportAPI
 from knowledge.models import KnowledgeScope
 from knowledge.serializers.common import get_knowledge_operation_object
 from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeBatchOperateSerializer
@@ -409,6 +410,68 @@ def get(self, request: Request, workspace_id: str, knowledge_id: str):
                 'workspace_id': workspace_id, 'knowledge_id': knowledge_id, 'user_id': request.user.id
             }).export_zip()
 
+    class ExportKnowledge(APIView):
+        authentication_classes = [TokenAuth]
+
+        @extend_schema(
+            summary=_('Export knowledge bundle'),
+            operation_id=_('Export knowledge bundle'),  # type: ignore
+            parameters=KnowledgeExportAPI.get_parameters(),
+            responses=KnowledgeExportAPI.get_response(),
+            tags=[_('Knowledge Base')]  # type: ignore
+        )
+        @has_permissions(
+            PermissionConstants.KNOWLEDGE_EXPORT.get_workspace_knowledge_permission(),
+            PermissionConstants.KNOWLEDGE_EXPORT.get_workspace_permission_workspace_manage_role(),
+            RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
+            ViewPermission([RoleConstants.USER.get_workspace_role()],
+                           [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
+        )
+        @log(
+            menu='Knowledge Base', operate="Export knowledge bundle",
+            get_operation_object=lambda r, keywords: get_knowledge_operation_object(keywords.get('knowledge_id')),
+        )
+        def get(self, request: Request, workspace_id: str, knowledge_id: str):
+            return KnowledgeSerializer.Operate(data={
+                'workspace_id': workspace_id, 'knowledge_id': knowledge_id, 'user_id': request.user.id
+            }).export_knowledge()
+
+
+    class ImportKnowledge(APIView):
+        authentication_classes = [TokenAuth]
+        parser_classes = [MultiPartParser]
+
+        @extend_schema(
+            methods=['POST'],
+            description=_('Import knowledge bundle'),
+            summary=_('Import knowledge bundle'),
+            operation_id=_('Import knowledge bundle'),
+            parameters=KnowledgeImportAPI.get_parameters(),
+            request=KnowledgeImportAPI.get_request(),
+            responses=KnowledgeImportAPI.get_response(),
+            tags=[_('Knowledge Base')]
+        )
+        @has_permissions(
+            PermissionConstants.KNOWLEDGE_CREATE.get_workspace_permission(),
+            RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
+            RoleConstants.USER.get_workspace_role()
+        )
+        @log(
+            menu='Knowledge Base', operate="Import knowledge bundle",
+        )
+        def post(self, request: Request, workspace_id: str):
+            is_import_tool = get_is_permissions(request, workspace_id=workspace_id)(
+                PermissionConstants.TOOL_IMPORT.get_workspace_permission(),
+                PermissionConstants.TOOL_IMPORT.get_workspace_permission_workspace_manage_role(),
+                RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), RoleConstants.USER.get_workspace_role()
+            )
+            return result.success(
+                KnowledgeSerializer.ImportKnowledge(
+                    data={'workspace_id': workspace_id, 'user_id': request.user.id, 'folder_id': request.data.get('folder_id',workspace_id)}
+                ).import_knowledge(request.FILES.get('file'), is_import_tool)
+            )
+
+
     class GenerateRelated(APIView):
         authentication_classes = [TokenAuth]
 

apps/knowledge/views/knowledge.py

@@ -9191,3 +9191,38 @@ msgstr "Batch move tools"
 msgid "Model is not allowed to be empty"
 msgstr ""
 
+msgid "Not a valid zip file"
+msgstr "Not a valid zip file"
+
+msgid "Not a valid KB export file, missing knowledge.json"
+msgstr "Not a valid KB export file, missing knowledge.json"
+
+msgid "Not a valid KB export file, missing knowledge.xlsx"
+msgstr "Not a valid KB export file, missing knowledge.xlsx"
+
+msgid "Tags"
+msgstr "Tags"
+
+msgid "Hit handling method"
+msgstr "Hit handling method"
+
+msgid "Directly return similarity"
+msgstr "Directly return similarity"
+
+msgid "Paragraph is active"
+msgstr "Paragraph is active"
+
+msgid "Document type"
+msgstr "Document type"
+
+msgid "Document meta"
+msgstr "Document meta"
+
+msgid "Export knowledge bundle"
+msgstr "Export knowledge bundle"
+
+msgid "Import knowledge bundle"
+msgstr "Import knowledge bundle"
+
+
+

apps/locales/en_US/LC_MESSAGES/django.po

@@ -9314,3 +9314,36 @@ msgstr "批量移动工具"
 msgid "Model is not allowed to be empty"
 msgstr "模型不能为空"
 
+msgid "Not a valid zip file"
+msgstr "不是有效的 zip 文件"
+
+msgid "Not a valid KB export file, missing knowledge.json"
+msgstr "不是有效的知识库导出文件，缺少 knowledge.json"
+
+msgid "Not a valid KB export file, missing knowledge.xlsx"
+msgstr "不是有效的知识库导出文件，缺少 knowledge.xlsx"
+
+msgid "Tags"
+msgstr "标签"
+
+msgid "Hit handling method"
+msgstr "命中处理方式"
+
+msgid "Directly return similarity"
+msgstr "直接返回相似度"
+
+msgid "Paragraph is active"
+msgstr "分段是否启用"
+
+msgid "Document type"
+msgstr "文档类型"
+
+msgid "Document meta"
+msgstr "文档元数据"
+
+msgid "Export knowledge bundle"
+msgstr "导出知识库"
+
+msgid "Import knowledge bundle"
+msgstr "导入知识库"
+

apps/locales/zh_CN/LC_MESSAGES/django.po

@@ -9311,3 +9311,36 @@ msgstr "批量移動工具"
 msgid "Model is not allowed to be empty"
 msgstr "模型不能为空"
 
+msgid "Not a valid zip file"
+msgstr "不是有效的 zip 檔案"
+
+msgid "Not a valid KB export file, missing knowledge.json"
+msgstr "不是有效的知識庫匯出檔案，缺少 knowledge.json"
+
+msgid "Not a valid KB export file, missing knowledge.xlsx"
+msgstr "不是有效的知識庫匯出檔案，缺少 knowledge.xlsx"
+
+msgid "Tags"
+msgstr "標籤"
+
+msgid "Hit handling method"
+msgstr "命中處理方式"
+
+msgid "Directly return similarity"
+msgstr "直接回傳相似度"
+
+msgid "Paragraph is active"
+msgstr "段落是否啟用"
+
+msgid "Document type"
+msgstr "文件類型"
+
+msgid "Document meta"
+msgstr "文件元資料"
+
+msgid "Export knowledge bundle"
+msgstr "匯出知識庫"
+
+msgid "Import knowledge bundle"
+msgstr "匯入知識庫"
+

apps/locales/zh_Hant/LC_MESSAGES/django.po

@@ -494,6 +494,39 @@ const postTransformWorkflow: (
   return post(`${prefix.value}/${knowledge_id}/transform_workflow`, data, undefined, loading)
 }
 
+/**
+ * 导出知识库
+ * @param knowledge_name 
+ * @param knowledge_id 
+ * @param loading 
+ * @returns 
+ */
+const exportKnowledgeBundle: (
+  knowledge_name: string,
+  knowledge_id: string,
+  loading?: Ref<boolean>
+) => Promise<any> = (knowledge_name, knowledge_id, loading) => {
+  return exportFile(
+    knowledge_name + '.zip',
+    `${prefix.value}/${knowledge_id}/export_knowledge`,undefined, loading
+  )
+}
+
+/**
+ * 导入知识库
+ * @param data 
+ * @param loading 
+ * @returns 
+ */
+const importKnowledgeBundle: (
+  data: any,
+  loading: Ref<boolean>
+) => Promise<Result<any>> = (data, loading) => {
+  return post(`${prefix.value}/import_knowledge`, data, undefined, loading)
+}
+
+
+
 
 export default {
   getKnowledgeList,
@@ -534,4 +567,6 @@ export default {
   exportKnowledgeWorkflow,
   importKnowledgeWorkflow,
   postTransformWorkflow,
+  exportKnowledgeBundle,
+  importKnowledgeBundle
 }

ui/src/api/knowledge/knowledge.ts

@@ -151,4 +151,5 @@ export default {
     subTitle: 'View Execution Record',
   },
   sourceType: 'Source type',
+  knowledgeImportTip: 'Knowledge base imported successfully. Documents have not been vectorized yet. Please configure the embedding model and vectorize the documents.'
 }

ui/src/locales/lang/en-US/common.ts

@@ -151,4 +151,5 @@ export default {
     subTitle: '查看执行记录',
   },
   sourceType: '资源类型',
+  knowledgeImportTip: '导入创建知识库成功，文档数据未向量化，请先设置知识库的向量模型，并对文档进行向量化操作'
 }