Skip to content

Commit c5935cf

Browse files
author
Jicheng Lu
committed
add knowledge base dictionary
1 parent dea50d4 commit c5935cf

13 files changed

Lines changed: 598 additions & 12 deletions

File tree

‎src/lib/common/Breadcrumb.svelte‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import { Row, Col, Breadcrumb, BreadcrumbItem } from '@sveltestrap/sveltestrap';
33
import Link from 'svelte-link';
44
5-
export let title = '';
5+
export let title = '';
66
export let pagetitle = '';
77
</script>
88

‎src/lib/helpers/http.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ function skipLoader(config) {
184184
new RegExp('http(s*)://(.*?)/knowledge/(.*?)/search', 'g'),
185185
new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'),
186186
new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'),
187+
new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'),
187188
new RegExp('http(s*)://(.*?)/users', 'g'),
188189
new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'),
189190
new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'),
@@ -227,6 +228,8 @@ function skipLoader(config) {
227228
new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'),
228229
new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'),
229230
new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'),
231+
new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'),
232+
new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'),
230233
new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'),
231234
new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'),
232235
new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'),

‎src/lib/helpers/types/commonTypes.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,4 +82,10 @@
8282
* @property {any[]} fail
8383
*/
8484

85+
/**
86+
* @typedef {Object} TableItemConfig
87+
* @property {string} dataName
88+
* @property {string} displayName
89+
*/
90+
8591
export default {};

‎src/lib/helpers/types/knowledgeTypes.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,4 +160,33 @@
160160
* @property {string} field_schema_type
161161
*/
162162

163+
/**
164+
* @typedef {Object} TokenizeRequest
165+
* @property {string} text
166+
* @property {string?} [provider]
167+
* @property {TokenizeOptions?} [options]
168+
*/
169+
170+
/**
171+
* @typedef {Object} TokenizeOptions
172+
* @property {string[]?} [data_providers]
173+
* @property {number?} [max_ngram]
174+
* @property {number?} [cutoff]
175+
* @property {number?} [top_k]
176+
*/
177+
178+
/**
179+
* @typedef {Object} TokenizeResponse
180+
* @property {TokenizeResult[]} [results]
181+
* @property {boolean?} [success]
182+
* @property {string?} [error_message]
183+
*/
184+
185+
/**
186+
* @typedef {Object} TokenizeResult
187+
* @property {string} token
188+
* @property {string?} [canonical_text]
189+
* @property {any} data
190+
*/
191+
163192
export default {};

‎src/lib/helpers/utils/common.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,11 +161,12 @@ export function scrollToBottom(container, behavior = 'smooth') {
161161

162162
/**
163163
* @param {string} str
164+
* @param {string} separator
164165
*/
165-
export function splitTextByCase(str) {
166+
export function splitTextByCase(str, separator = '_') {
166167
if (!str) return str;
167168

168-
let words = str.split("_");
169+
let words = str.split(separator);
169170
if (words.length === 1) {
170171
// split by camel case
171172
words = str.split(/(?=[A-Z])/);

‎src/lib/scss/custom/pages/_knowledgebase.scss‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -288,10 +288,11 @@
288288
display: flex;
289289
gap: 5px;
290290
justify-content: flex-end;
291-
min-width: 300px;
291+
flex-wrap: wrap;
292292

293293
.collection-dropdown {
294-
width: 100%;
294+
flex: 1 1 auto;
295+
min-width: 150px;
295296
}
296297

297298
.collection-action-btn {
@@ -304,6 +305,17 @@
304305
outline: none !important;
305306
box-shadow: none !important;
306307
}
308+
309+
@media (max-width: 350px) {
310+
flex-direction: column;
311+
justify-content: center;
312+
gap: 5px;
313+
314+
.collection-dropdown {
315+
width: 100%;
316+
min-width: 100%;
317+
}
318+
}
307319
}
308320

309321
.collection-add-container {
@@ -385,15 +397,19 @@
385397

386398
.more-detail-list {
387399
margin: 1px 0px 15px 0px;
388-
font-size: 10px;
400+
font-size: 12px;
389401
padding-left: 4rem;
390402
overflow-y: auto;
391403
scrollbar-width: none;
392404
max-height: 200px;
393405

394-
.more-dtail-item {
406+
.more-detail-item {
395407
list-style-type: square;
396408
}
409+
410+
ul {
411+
font-size: 15px;
412+
}
397413
}
398414
}
399415
}

‎src/lib/services/api-endpoints.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,10 @@ export const endpoints = {
114114
knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`,
115115
knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`,
116116

117+
tokenizersUrl: `${host}/knowledge/tokenizer/providers`,
118+
tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`,
119+
tokenizeUrl: `${host}/knowledge/tokenize`,
120+
117121
// chathub
118122
chatHubUrl: `${host}/chatHub`,
119123

‎src/lib/services/knowledge-base-service.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,4 +274,37 @@ export async function deleteVectorIndexes(collection, options) {
274274
}
275275
});
276276
return response.data;
277+
}
278+
279+
280+
281+
282+
/**
283+
* @returns {Promise<string[]>}
284+
*/
285+
export async function getTokenizers() {
286+
const url = endpoints.tokenizersUrl;
287+
const response = await axios.get(url);
288+
return response.data;
289+
}
290+
291+
/**
292+
* @returns {Promise<string[]>}
293+
*/
294+
export async function getTokenizerDataLoaders() {
295+
const url = endpoints.tokenizerDataLoadersUrl;
296+
const response = await axios.get(url);
297+
return response.data;
298+
}
299+
300+
/**
301+
* @param {import('$knowledgeTypes').TokenizeRequest} request
302+
* @returns {Promise<import('$knowledgeTypes').TokenizeResponse>}
303+
*/
304+
export async function tokenize(request) {
305+
const url = endpoints.tokenizeUrl;
306+
const response = await axios.post(url, {
307+
...request
308+
});
309+
return response.data;
277310
}

‎src/routes/page/knowledge-base/common/table/table-item.svelte‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
<script>
2+
import { Button } from "@sveltestrap/sveltestrap";
3+
import { fly } from 'svelte/transition';
4+
import Loader from "$lib/common/Loader.svelte";
5+
import JSONTree from 'svelte-json-tree';
6+
import { formatObject, splitTextByCase } from "$lib/helpers/utils/common";
7+
8+
/** @type {any} */
9+
export let item;
10+
11+
/** @type {import('$commonTypes').TableItemConfig[]} */
12+
export let columns;
13+
14+
/** @type {string | null} */
15+
export let detailKey = null;
16+
17+
/** @type {boolean} */
18+
export let useJsonDisplay = false;
19+
20+
/** @type {boolean} */
21+
export let open = false;
22+
23+
let isLoading = false;
24+
let loadMore = false;
25+
26+
$: {
27+
if (!open) {
28+
loadMore = false;
29+
}
30+
}
31+
32+
function toggleDetail() {
33+
open = !open;
34+
}
35+
</script>
36+
37+
{#if isLoading}
38+
<Loader />
39+
{/if}
40+
41+
<tr in:fly={{ y: -5, duration: 800 }}>
42+
{#if columns?.length > 0}
43+
{#each columns as column, idx (idx)}
44+
<td style={`max-width: 50px; width: ${80 / columns.length}%;`}>
45+
<div class="ellipsis">{item[column.dataName] || ''}</div>
46+
</td>
47+
{/each}
48+
{/if}
49+
<td class="knowledge-op">
50+
<ul class="list-unstyled hstack gap-1 mb-0 knowledge-op-list">
51+
{#if detailKey}
52+
<li data-bs-toggle="tooltip" data-bs-placement="top" title="View Detail">
53+
<Button
54+
class="btn btn-sm btn-soft-primary"
55+
on:click={() => toggleDetail()}
56+
>
57+
{#if open}
58+
<i class="bx bx-hide" />
59+
{:else}
60+
<i class="mdi mdi-eye-outline" />
61+
{/if}
62+
</Button>
63+
</li>
64+
{/if}
65+
</ul>
66+
</td>
67+
</tr>
68+
69+
{#if open}
70+
<tr in:fly={{ y: -5, duration: 800 }} out:fly={{ y: -5, duration: 300 }}>
71+
<td colspan="12">
72+
<div class="knowledge-detail">
73+
<ul>
74+
{#each columns as column, idx (idx)}
75+
<li>
76+
<div class="wrappable fw-bold text-primary">
77+
{column.displayName || column.dataName}
78+
</div>
79+
<div class="wrappable">
80+
{item[column.dataName] || ''}
81+
</div>
82+
</li>
83+
{/each}
84+
</ul>
85+
<div class="more-detail">
86+
<Button class='toggle-btn btn-sm' color="link" on:click={() => loadMore = !loadMore}>
87+
{`${loadMore ? 'Less -' : 'More +'}`}
88+
</Button>
89+
</div>
90+
{#if loadMore}
91+
<ul
92+
class="more-detail-list text-secondary"
93+
in:fly={{ y: -5, duration: 300 }}
94+
out:fly={{ y: -5, duration: 200 }}
95+
>
96+
{#if detailKey && item[detailKey]}
97+
{#if useJsonDisplay}
98+
<JSONTree
99+
value={formatObject(item[detailKey])}
100+
defaultExpandedLevel={1}
101+
--json-tree-number-color="var(--bs-info)"
102+
--json-tree-boolean-color="var(--bs-info)"
103+
--json-tree-string-color="var(--bs-info)"
104+
/>
105+
{:else}
106+
{#each Object.keys(item[detailKey]) as key, idx (idx)}
107+
<li class="more-detail-item wrappable">
108+
<span>{splitTextByCase(key)}: </span>
109+
<span>{item[detailKey][key]}</span>
110+
</li>
111+
{/each}
112+
{/if}
113+
{/if}
114+
</ul>
115+
{/if}
116+
</div>
117+
</td>
118+
</tr>
119+
{/if}

0 commit comments

Comments
 (0)