refine doc knowledge

Author: Jicheng Lu

src/lib/helpers/http.js

@@ -184,6 +184,7 @@ function skipLoader(config) {
         new RegExp('http(s*)://(.*?)/knowledge/(.*?)/search', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/upload', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'),
         new RegExp('http(s*)://(.*?)/users', 'g'),
         new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'),
@@ -228,6 +229,7 @@ function skipLoader(config) {
         new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/document/processors', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'),
         new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'),

src/lib/helpers/types/knowledgeTypes.js

@@ -98,6 +98,7 @@
 /**
  * @typedef {Object} VectorKnowledgeUploadRequest
  * @property {import('$fileTypes').FileModel[]} files - The files.
+ * @property {any} [options]
  */
 
 /**

src/lib/scss/custom/pages/_knowledgebase.scss

@@ -482,7 +482,7 @@
     }
 
     .doc-upload-footer {
-        margin-top: 30px;
+        margin-top: 0px;
 
         .load-doc-btn {
             font-size: 12px;
@@ -820,4 +820,25 @@
             gap: 0;
         }
     }
+}
+
+
+.knowledge-upload-modal-container {
+    .doc-upload-body {
+        width: 100%;
+        height: 100%;
+
+        .gallery-item, .doc-drop-zone {
+            width: 100%;
+            min-height: 200px;
+        }
+
+        i {
+            font-size: 5em;
+        }
+
+        .doc-drop-icon {
+            text-align: center;
+        }
+    }
 }

src/lib/services/api-endpoints.js

@@ -113,6 +113,7 @@ export const endpoints = {
     knowledgeDocumentDeleteUrl: `${host}/knowledge/document/{collection}/delete/{fileId}`,
     knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`,
     knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`,
+    knowledgeDocumentProcessorsUrl: `${host}/knowledge/document/processors`,
 
     tokenizersUrl: `${host}/knowledge/tokenizer/providers`,
     tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`,

src/lib/services/knowledge-base-service.js

@@ -193,6 +193,15 @@ export async function getKnowledgeDocumentPageList(collection, request) {
     return response.data;
 }
 
+/**
+ * @returns {Promise<string[]>}
+ */
+export async function getKnowledgeDocumentProcessors() {
+    const url = endpoints.knowledgeDocumentProcessorsUrl;
+    const response = await axios.get(url);
+    return response.data;
+}
+
 
 /**
  * @param {import('$knowledgeTypes').CreateVectorCollectionRequest} request

src/routes/page/knowledge-base/documents/+page.svelte

@@ -25,9 +25,8 @@
 		createVectorCollection,
 		createVectorIndexes,
 		deleteVectorIndexes,
-
-		getVectorCollectionDetails
-
+		getVectorCollectionDetails,
+		getKnowledgeDocumentProcessors
     } from '$lib/services/knowledge-base-service';
 	import Breadcrumb from '$lib/common/Breadcrumb.svelte';
     import HeadTitle from '$lib/common/HeadTitle.svelte';
@@ -41,13 +40,14 @@
 		VectorDataSource,
 		VectorPayloadDataType
 	} from '$lib/helpers/enums';
+	import { DECIMAL_REGEX } from '$lib/helpers/constants';
 	import VectorItem from '../common/vector-table/vector-item.svelte';
 	import VectorItemEditModal from '../common/vector-table/vector-item-edit-modal.svelte';
 	import CollectionCreateModal from '../common/collection/collection-create-modal.svelte';
 	import VectorIndexModal from '../common/indexes/vector-index-modal.svelte';
 	import AdvancedSearch from '../common/search/advanced-search.svelte';
 	import KnowledgeDocumentUpload from './knowledge-document-upload.svelte';
-	import { DECIMAL_REGEX } from '$lib/helpers/constants';
+	import KnowledgeUploadModal from './knowledge-upload-modal.svelte';
 	
 	const pageSize = 8;
   	const duration = 2000;
@@ -106,6 +106,9 @@
 		order: 'desc'
 	};
 
+	/** @type {import('$commonTypes').LabelValuePair[]} */
+	let docProcessors = [];
+
 	/** @type {boolean} */
     let showDemo = true;
     let isSearching = false;
@@ -122,6 +125,8 @@
 	let isAdvSearchOn = false;
 	let disableSearchBtn = false;
 	let isExactSearch = false;
+	let isOpenUploadModal = false;
+	let showDocList = false;
 
     /** @type {any} */
     let docUploadrCmp;
@@ -168,6 +173,7 @@
 			if (selectedCollection) {
 				Promise.all([
 					getCollectionDetail(),
+					getDocProcessors(),
 					getData({
 						...defaultParams,
 						isReset: true,
@@ -456,6 +462,19 @@
 		});
 	}
 
+	function getDocProcessors() {
+		return new Promise((resolve, reject) => {
+			getKnowledgeDocumentProcessors().then(res => {
+				const retProcessors = res?.map(x => ({  label: x, value: x })) || [];
+				docProcessors = [ ...retProcessors ];
+				resolve(res);
+			}).catch(err => {
+				docProcessors = [];
+				reject(err);
+			});
+		});
+	}
+
 	/** @param {boolean} isLocalLoading */
 	function toggleLoader(isLocalLoading) {
 		if (isLocalLoading) {
@@ -740,9 +759,12 @@
         });
 	}
 
-    /** @param {any} e */
-    function onDocUploaded(e) {
-        reset();
+    /**
+	 * @param {any} e
+	 * @param {boolean} skipLoader
+	 */
+    function onDocUploaded(e, skipLoader = false) {
+        reset(skipLoader);
     }
 
     /** @param {any} e */
@@ -859,6 +881,10 @@
 	function toggleIndexModal() {
 		isOpenIndexModal = !isOpenIndexModal;
 	}
+
+	function toggleUploadModal() {
+		isOpenUploadModal = !isOpenUploadModal;
+	}
 </script>
 
 <HeadTitle title="{$_('Document Knowledge')}" addOn="Knowledge Base" />
@@ -905,6 +931,21 @@
 	/>
 {/if}
 
+<KnowledgeUploadModal
+	className={'knowledge-upload-modal-container'}
+	size={'md'}
+	accept={'.txt,.pdf'}
+	collection={selectedCollection}
+	open={isOpenUploadModal}
+	processors={docProcessors}
+	toggleModal={() => toggleUploadModal()}
+	on:docuploaded={(e) => {
+		showDocList = false;
+		isOpenUploadModal = false;
+		onDocUploaded(e, true);
+	}}
+/>
+
 <CollectionCreateModal
 	title={'Create new collection'}
 	open={isOpenCreateCollection}
@@ -1082,9 +1123,13 @@
         {#if selectedCollection}
             <KnowledgeDocumentUpload
                 collection={selectedCollection}
+				accept={'.txt'}
+				disableFileUpload={true}
+				disableFileDelete={false}
                 disabled={disabled}
                 bind:this={docUploadrCmp}
-                on:docuploaded={(e) => onDocUploaded(e)}
+				bind:showDocList={showDocList}
+                on:docuploaded={(e) => onDocUploaded(e, true)}
                 on:docdeleted={(e) => onDocDeleted(e)}
 				on:resetdocs={(e) => onDocsReset(e)}
             />
@@ -1133,6 +1178,20 @@
 											<i class="mdi mdi-minus" />
 										</Button>
 									</div>
+									<div
+										class="line-align-center"
+										data-bs-toggle="tooltip"
+										data-bs-placement="top"
+										title="Upload doc"
+									>
+										<Button
+											class="btn btn-sm btn-soft-info knowledge-btn-icon"
+											disabled={disabled}
+											on:click={() => toggleUploadModal()}
+										>
+											<i class="mdi mdi-file-upload" />
+										</Button>
+									</div>
 								</div>
 								<div class="collection-action-container action-container-padding">
 									{#if selectedCollection}