Skip to content

Commit 49c1d2c

Browse files
iceljciceljc
authored
Merge pull request #407 from iceljc/features/refine-chat-window
Rename to NER
2 parents 7664ede + f0aa034 commit 49c1d2c

8 files changed

Lines changed: 99 additions & 65 deletions

File tree

‎src/lib/helpers/http.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ function skipLoader(config) {
185185
new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'),
186186
new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'),
187187
new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/upload', 'g'),
188-
new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'),
188+
new RegExp('http(s*)://(.*?)/knowledge/entity/analyze', 'g'),
189189
new RegExp('http(s*)://(.*?)/users', 'g'),
190190
new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'),
191191
new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'),
@@ -229,9 +229,9 @@ function skipLoader(config) {
229229
new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'),
230230
new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'),
231231
new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'),
232-
new RegExp('http(s*)://(.*?)/knowledge/document/processors', 'g'),
233-
new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'),
234-
new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'),
232+
new RegExp('http(s*)://(.*?)/knowledge/processors', 'g'),
233+
new RegExp('http(s*)://(.*?)/knowledge/entity/analyzers', 'g'),
234+
new RegExp('http(s*)://(.*?)/knowledge/entity/data-providers', 'g'),
235235
new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'),
236236
new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'),
237237
new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'),

‎src/lib/helpers/types/knowledgeTypes.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -162,29 +162,29 @@
162162
*/
163163

164164
/**
165-
* @typedef {Object} TokenizeRequest
165+
* @typedef {Object} EntityAnalysisRequest
166166
* @property {string} text
167167
* @property {string?} [provider]
168-
* @property {TokenizeOptions?} [options]
168+
* @property {EntityAnalysisOptions?} [options]
169169
*/
170170

171171
/**
172-
* @typedef {Object} TokenizeOptions
172+
* @typedef {Object} EntityAnalysisOptions
173173
* @property {string[]?} [data_providers]
174174
* @property {number?} [max_ngram]
175175
* @property {number?} [cutoff]
176176
* @property {number?} [top_k]
177177
*/
178178

179179
/**
180-
* @typedef {Object} TokenizeResponse
181-
* @property {TokenizeResult[]} [results]
180+
* @typedef {Object} EntityAnalysisResponse
181+
* @property {EntityAnalysisResult[]} [results]
182182
* @property {boolean?} [success]
183183
* @property {string?} [error_message]
184184
*/
185185

186186
/**
187-
* @typedef {Object} TokenizeResult
187+
* @typedef {Object} EntityAnalysisResult
188188
* @property {string} token
189189
* @property {string?} [canonical_text]
190190
* @property {any} data

‎src/lib/services/api-endpoints.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,11 @@ export const endpoints = {
113113
knowledgeDocumentDeleteUrl: `${host}/knowledge/document/{collection}/delete/{fileId}`,
114114
knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`,
115115
knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`,
116-
knowledgeDocumentProcessorsUrl: `${host}/knowledge/document/processors`,
116+
knowledgeProcessorsUrl: `${host}/knowledge/processors`,
117117

118-
tokenizersUrl: `${host}/knowledge/tokenizer/providers`,
119-
tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`,
120-
tokenizeUrl: `${host}/knowledge/tokenize`,
118+
entityAnalyzersUrl: `${host}/knowledge/entity/analyzers`,
119+
entityDataLoadersUrl: `${host}/knowledge/entity/data-providers`,
120+
entityAnalyzeUrl: `${host}/knowledge/entity/analyze`,
121121

122122
// chathub
123123
chatHubUrl: `${host}/chatHub`,

‎src/lib/services/knowledge-base-service.js‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -196,8 +196,8 @@ export async function getKnowledgeDocumentPageList(collection, request) {
196196
/**
197197
* @returns {Promise<string[]>}
198198
*/
199-
export async function getKnowledgeDocumentProcessors() {
200-
const url = endpoints.knowledgeDocumentProcessorsUrl;
199+
export async function getKnowledgeProcessors() {
200+
const url = endpoints.knowledgeProcessorsUrl;
201201
const response = await axios.get(url);
202202
return response.data;
203203
}
@@ -291,27 +291,27 @@ export async function deleteVectorIndexes(collection, options) {
291291
/**
292292
* @returns {Promise<string[]>}
293293
*/
294-
export async function getTokenizers() {
295-
const url = endpoints.tokenizersUrl;
294+
export async function getEntityAnalyzers() {
295+
const url = endpoints.entityAnalyzersUrl;
296296
const response = await axios.get(url);
297297
return response.data;
298298
}
299299

300300
/**
301301
* @returns {Promise<string[]>}
302302
*/
303-
export async function getTokenizerDataLoaders() {
304-
const url = endpoints.tokenizerDataLoadersUrl;
303+
export async function getEntityDataLoaders() {
304+
const url = endpoints.entityDataLoadersUrl;
305305
const response = await axios.get(url);
306306
return response.data;
307307
}
308308

309309
/**
310-
* @param {import('$knowledgeTypes').TokenizeRequest} request
311-
* @returns {Promise<import('$knowledgeTypes').TokenizeResponse>}
310+
* @param {import('$knowledgeTypes').EntityAnalysisRequest} request
311+
* @returns {Promise<import('$knowledgeTypes').EntityAnalysisResponse>}
312312
*/
313-
export async function tokenize(request) {
314-
const url = endpoints.tokenizeUrl;
313+
export async function analyzeEntity(request) {
314+
const url = endpoints.entityAnalyzeUrl;
315315
const response = await axios.post(url, {
316316
...request
317317
});

‎src/routes/page/agent/[agentId]/agent-components/agent-instruction.svelte‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,22 @@
8787
handleAgentChange();
8888
}
8989
90+
/** @param {any} e */
91+
function onKeyDown(e) {
92+
if (e.key === 'Tab') {
93+
e.preventDefault();
94+
95+
if (e.target) {
96+
const start = e.target.selectionStart || 0;
97+
const end = e.target.selectionEnd || 0;
98+
const value = e.target.value || '';
99+
e.target.value = value.substring(0, start) + "\t" + value.substring(end);
100+
e.target.selectionStart = start + 1;
101+
e.target.selectionEnd = start + 1;
102+
}
103+
}
104+
}
105+
90106
function addChannel() {
91107
inner_instructions = [
92108
...inner_instructions,
@@ -193,6 +209,7 @@
193209
value={selected_instruction.instruction}
194210
rows={20}
195211
on:input={(e) => changePrompt(e)}
212+
on:keydown={(e) => onKeyDown(e)}
196213
placeholder="Enter your instruction"
197214
/>
198215
</FormGroup>

‎src/routes/page/agent/[agentId]/agent-components/agent-template.svelte‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,22 @@
8080
handleAgentChange();
8181
}
8282
83+
/** @param {any} e */
84+
function onKeyDown(e) {
85+
if (e.key === 'Tab') {
86+
e.preventDefault();
87+
88+
if (e.target) {
89+
const start = e.target.selectionStart || 0;
90+
const end = e.target.selectionEnd || 0;
91+
const value = e.target.value || '';
92+
e.target.value = value.substring(0, start) + "\t" + value.substring(end);
93+
e.target.selectionStart = start + 1;
94+
e.target.selectionEnd = start + 1;
95+
}
96+
}
97+
}
98+
8399
function addTemplate() {
84100
inner_templates = [
85101
...inner_templates,
@@ -164,10 +180,11 @@
164180
type="textarea"
165181
class="form-control"
166182
style="scrollbar-width: thin; resize: none;"
183+
placeholder="Enter your content"
167184
value={selected_template.content}
168185
rows={15}
169186
on:input={(e) => changePrompt(e)}
170-
placeholder="Enter your content"
187+
on:keydown={(e) => onKeyDown(e)}
171188
/>
172189
{/if}
173190
</FormGroup>

‎src/routes/page/knowledge-base/dictionary/+page.svelte‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
Table
1111
} from '@sveltestrap/sveltestrap';
1212
import {
13-
getTokenizers,
14-
getTokenizerDataLoaders,
15-
tokenize
13+
getEntityAnalyzers,
14+
getEntityDataLoaders,
15+
analyzeEntity
1616
} from '$lib/services/knowledge-base-service';
1717
import Breadcrumb from '$lib/common/Breadcrumb.svelte';
1818
import HeadTitle from '$lib/common/HeadTitle.svelte';
@@ -40,19 +40,19 @@
4040
let elapsedTime = '';
4141
4242
/** @type {string | null} */
43-
let selectedTokenizer = null;
43+
let selectedAnalyzer = null;
4444
4545
/** @type {string[]} */
4646
let selectedDataLoaders = [];
4747
48-
/** @type {import('$knowledgeTypes').TokenizeResult[]} */
48+
/** @type {import('$knowledgeTypes').EntityAnalysisResult[]} */
4949
let items = [];
5050
5151
/** @type {import('$commonTypes').LabelValuePair[]} */
52-
let tokenizers = [];
52+
let analyzers = [];
5353
5454
/** @type {import('$commonTypes').LabelValuePair[]} */
55-
let tokenizerDataLoaders = [];
55+
let dataLoaders = [];
5656
5757
/** @type {number | null | undefined} */
5858
let totalDataCount;
@@ -68,7 +68,7 @@
6868
6969
$: {
7070
disableSearchBtn = false;
71-
if (!selectedTokenizer || isSearching) {
71+
if (!selectedAnalyzer || isSearching) {
7272
disableSearchBtn = true;
7373
} else if (!text || util.trim(text).length === 0) {
7474
disableSearchBtn = true;
@@ -82,8 +82,8 @@
8282
function initData() {
8383
isLoading = true;
8484
Promise.all([
85-
getTokenizerProviders(),
86-
getTokenizerDataLoaderProviders()
85+
getAnalyzerProviders(),
86+
getDataLoaderProviders()
8787
]).finally(() => {
8888
isLoading = false;
8989
});
@@ -102,7 +102,7 @@
102102
elapsedTime = '';
103103
const start = new Date();
104104
105-
getTokenizeResult().finally(() => {
105+
getAnalysisResult().finally(() => {
106106
isSearching = false;
107107
searchDone = true;
108108
const gap = new Date().getTime() - start.getTime();
@@ -137,45 +137,45 @@
137137
selectedDataLoaders = [];
138138
}
139139
140-
function getTokenizerProviders() {
140+
function getAnalyzerProviders() {
141141
return new Promise((resolve, reject) => {
142-
getTokenizers().then(res => {
142+
getEntityAnalyzers().then(res => {
143143
const retProviders = res?.map(x => ({ label: x, value: x })) || [];
144-
tokenizers = [ ...retProviders ];
145-
selectedTokenizer = tokenizers[0]?.value;
144+
analyzers = [ ...retProviders ];
145+
selectedAnalyzer = analyzers[0]?.value;
146146
resolve(res);
147147
}).catch(err => {
148-
tokenizers = [];
149-
selectedTokenizer = tokenizers[0]?.value;
148+
analyzers = [];
149+
selectedAnalyzer = null;
150150
reject(err);
151151
});
152152
});
153153
}
154154
155-
function getTokenizerDataLoaderProviders() {
155+
function getDataLoaderProviders() {
156156
return new Promise((resolve, reject) => {
157-
getTokenizerDataLoaders().then(res => {
157+
getEntityDataLoaders().then(res => {
158158
const retProviders = res?.map(x => ({ label: x, value: x })) || [];
159-
tokenizerDataLoaders = [ ...retProviders ];
159+
dataLoaders = [ ...retProviders ];
160160
resolve(res);
161161
}).catch(err => {
162-
tokenizerDataLoaders = [];
162+
dataLoaders = [];
163163
reject(err);
164164
});
165165
});
166166
}
167167
168-
function getTokenizeResult() {
168+
function getAnalysisResult() {
169169
return new Promise((resolve, reject) => {
170170
const request = {
171171
text: util.trim(text),
172-
provider: selectedTokenizer,
172+
provider: selectedAnalyzer,
173173
options: {
174174
data_providers: selectedDataLoaders?.length > 0 ? selectedDataLoaders : null
175175
}
176176
};
177177
178-
tokenize(request).then(res => {
178+
analyzeEntity(request).then(res => {
179179
items = res?.results || [];
180180
totalDataCount = items.length;
181181
resolve(res);
@@ -188,13 +188,13 @@
188188
}
189189
190190
/** @param {any} e */
191-
function changeTokenizer(e) {
191+
function changeAnalyzer(e) {
192192
const selectedValues = e?.detail?.selecteds || [];
193-
selectedTokenizer = selectedValues[0]?.value;
193+
selectedAnalyzer = selectedValues[0]?.value;
194194
}
195195
196196
/** @param {any} e */
197-
function changeTokenizerDataLoaders(e) {
197+
function changeDataLoaders(e) {
198198
const selectedValues = e?.detail?.selecteds || [];
199199
// @ts-ignore
200200
selectedDataLoaders = selectedValues.map(x => x.value);
@@ -314,27 +314,27 @@
314314
<div class="collection-action-container action-container-padding">
315315
<div class="collection-dropdown-container">
316316
<div class="line-align-center collection-dropdown">
317-
<div>Tokenizer</div>
317+
<div>Analyzer</div>
318318
<Select
319-
tag={'tokenizer-select'}
320-
placeholder={'Select Tokenizer'}
319+
tag={'entity-analyzer-select'}
320+
placeholder={'Select Analyzer'}
321321
searchMode
322-
selectedValues={selectedTokenizer ? [selectedTokenizer] : []}
323-
options={tokenizers}
324-
on:select={e => changeTokenizer(e)}
322+
selectedValues={selectedAnalyzer ? [selectedAnalyzer] : []}
323+
options={analyzers}
324+
on:select={e => changeAnalyzer(e)}
325325
/>
326326
</div>
327327
<div class="line-align-center collection-dropdown">
328-
<div>Data Loaders</div>
328+
<div>Data Providers</div>
329329
<Select
330-
tag={'tokenizer-data-loader-select'}
331-
placeholder={'Select Data Loaders'}
330+
tag={'entity-data-loader-select'}
331+
placeholder={'Select Data Providers'}
332332
searchMode
333333
selectAll
334334
multiSelect
335335
selectedValues={selectedDataLoaders}
336-
options={tokenizerDataLoaders}
337-
on:select={e => changeTokenizerDataLoaders(e)}
336+
options={dataLoaders}
337+
on:select={e => changeDataLoaders(e)}
338338
/>
339339
</div>
340340
</div>

‎src/routes/page/knowledge-base/documents/+page.svelte‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
createVectorIndexes,
2727
deleteVectorIndexes,
2828
getVectorCollectionDetails,
29-
getKnowledgeDocumentProcessors
29+
getKnowledgeProcessors
3030
} from '$lib/services/knowledge-base-service';
3131
import Breadcrumb from '$lib/common/Breadcrumb.svelte';
3232
import HeadTitle from '$lib/common/HeadTitle.svelte';
@@ -464,7 +464,7 @@
464464
465465
function getDocProcessors() {
466466
return new Promise((resolve, reject) => {
467-
getKnowledgeDocumentProcessors().then(res => {
467+
getKnowledgeProcessors().then(res => {
468468
const retProcessors = res?.map(x => ({ label: x, value: x })) || [];
469469
docProcessors = [ ...retProcessors ];
470470
resolve(res);

0 commit comments

Comments
 (0)