refine llm configs

Author: Jicheng Lu

src/lib/helpers/enums.js

@@ -203,14 +203,34 @@ const globalEvent = {
 export const GlobalEvent = Object.freeze(globalEvent);
 
 const llmModelType = {
-    Text: 1,
-    Chat: 2,
-    Image: 3,
-    Embedding: 4,
-    Audio: 5
+    All: "All",
+    Text: "Text",
+    Chat: "Chat",
+    Image: "Image",
+    Embedding: "Embedding",
+    Audio: "Audio",
+    Realtime: "Realtime",
+    Web: "Web"
 };
 export const LlmModelType = Object.freeze(llmModelType);
 
+const llmModelCapability = {
+    All: "All",
+    Text: "Text",
+    Chat: "Chat",
+    ImageReading: "ImageReading",
+    ImageGeneration: "ImageGeneration",
+    ImageEdit: "ImageEdit",
+    ImageVariation: "ImageVariation",
+    Embedding: "Embedding",
+    AudioTranscription: "AudioTranscription",
+    AudioGeneration: "AudioGeneration",
+    Realtime: "Realtime",
+    WebSearch: "WebSearch",
+    PdfReading: "PdfReading"
+};
+export const LlmModelCapability = Object.freeze(llmModelCapability);
+
 const reasoningEffortLevel = {
     Minimal: "minimal",
     Low: "low",

src/lib/helpers/types/agentTypes.js

@@ -18,6 +18,10 @@
  * @property {number} max_recursion_depth
  * @property {number?} [max_output_tokens]
  * @property {string?} [reasoning_effort_level]
+ * @property {any} [image_generation]
+ * @property {any} [image_edit]
+ * @property {any} [audio_transcription]
+ * @property {any} [realtime]
  */
 
 

src/lib/helpers/types/commonTypes.js

@@ -39,10 +39,13 @@
  */
 
 /**
- * @typedef {Object} LlmConfigOption
- * @property {number?} [type]
+ * @typedef {Object} LlmConfigFilter
+ * @property {string[]?} [providers]
+ * @property {string[]?} [modelIds]
+ * @property {string[]?} [modelNames]
+ * @property {string[]?} [modelTypes]
+ * @property {string[]?} [modelCapabilities]
  * @property {boolean?} [multiModal]
- * @property {boolean?} [imageGeneration]
  */
 
 /**
@@ -55,6 +58,8 @@
  * @typedef {Object} LlmModelSetting
  * @property {string} name
  * @property {string} type
+ * @property {string[]} capabilities
+ * @property {boolean} multiModal
  * @property {any} reasoning
  */
 

src/lib/scss/custom/pages/_agent.scss

@@ -143,13 +143,20 @@
     }
 }
 
+.agent-config-container {
+    padding: 10px;
+    border: 1px dashed var(--bs-primary);
+    border-radius: 5px;
+}
+
 .agent-utility-container {
     display: flex;
     flex-direction: column;
     gap: 10px;
     max-height: 500px;
     overflow-y: auto;
     scrollbar-width: thin;
+    padding: 0px 10px;
 
     .merge-utility {
         display: flex;

src/lib/services/llm-provider-service.js

@@ -1,6 +1,7 @@
 import { endpoints } from './api-endpoints.js';
 import { replaceUrl } from '$lib/helpers/http.js';
 import axios from 'axios';
+import qs from 'qs';
 
 /**
  * Get provider list
@@ -26,19 +27,15 @@ export async function getLlmProviderModels(provider) {
 
 /**
  * Get llm configs
- * @param {import('$commonTypes').LlmConfigOption?} [options]
+ * @param {import('$commonTypes').LlmConfigFilter?} [filter]
  * @returns {Promise<import('$commonTypes').LlmConfig[]>}
  */
-export async function getLlmConfigs(options = null) {
+export async function getLlmConfigs(filter = null) {
     const url = endpoints.llmConfigsUrl;
+    const params = filter != null ? { filter: filter } : null;
     const response = await axios.get(url, {
-        params: {
-            options: options
-        },
-        paramsSerializer: {
-            dots: true,
-            indexes: null,
-        }
+        params: params,
+        paramsSerializer: (params) => qs.stringify(params, { encode: false, allowDots: true, arrayFormat: "indices" })
     });
     return response.data;
 }

src/routes/page/agent/[agentId]/agent-components/agent-llm-config.svelte

@@ -1,9 +1,12 @@
 <script>
     import { onMount } from 'svelte';
-    import { Card, CardBody, Input } from '@sveltestrap/sveltestrap';
-    import { getLlmProviders, getLlmProviderModels } from '$lib/services/llm-provider-service';
-	import { INTEGER_REGEX } from '$lib/helpers/constants';
-	import { ReasoningEffortLevel } from '$lib/helpers/enums';
+    import { Card, CardBody } from '@sveltestrap/sveltestrap';
+    import { getLlmConfigs } from '$lib/services/llm-provider-service';
+	import ChatConfig from './llm-configs/chat-config.svelte';
+	import ImageGenerationConfig from './llm-configs/image-generation-config.svelte';
+	import ImageEditConfig from './llm-configs/image-edit-config.svelte';
+	import AudioTranscriptionConfig from './llm-configs/audio-transcription-config.svelte';
+	import RealtimeConfig from './llm-configs/realtime-config.svelte';
     
     /** @type {import('$agentTypes').AgentModel} */
     export let agent;
@@ -12,195 +15,57 @@
     export let handleAgentChange = () => {};
 
     export const fetchLlmConfig = () => {
-        const reasoningEffort = models.find(x => x.name === config.model)?.reasoning != null ? config.reasoning_effort_level : null;
+        const chatConfig = chatConfigCmp?.fetchConfig();
+        const imageGenerationConfig = imageGenerationConfigCmp?.fetchConfig();
+        const imageEditConfig = imageEditConfigCmp?.fetchConfig();
+        const audioTranscriptionConfig = audioTranscriptionConfigCmp?.fetchConfig();
+        const realtimeConfig = realtimeConfigCmp?.fetchConfig();
         return {
-            ...config,
-            max_output_tokens: Number(config.max_output_tokens) > 0 ? Number(config.max_output_tokens) : null,
-            reasoning_effort_level: reasoningEffort
+            ...chatConfig,
+            image_generation: imageGenerationConfig ? {...imageGenerationConfig} : null,
+            image_edit: imageEditConfig ? {...imageEditConfig} : null,
+            audio_transcription: audioTranscriptionConfig ? {...audioTranscriptionConfig} : null,
+            realtime: realtimeConfig ? {...realtimeConfig} : null
         };
     }
 
-    const recursiveDepthLowerLimit = 1;
+    /** @type {any} */
+    let chatConfigCmp;
+    /** @type {any} */
+    let imageGenerationConfigCmp;
+    /** @type {any} */
+    let imageEditConfigCmp;
+    /** @type {any} */
+    let audioTranscriptionConfigCmp;
+    /** @type {any} */
+    let realtimeConfigCmp;
     
-    /** @type {import('$commonTypes').LabelValuePair[]} */
-	const reasonLevelOptions = [
-        { value: '', label: '' },
-        ...Object.entries(ReasoningEffortLevel).map(([k, v]) => ({
-            value: v,
-            label: v
-        }))
-    ];
-
-    let config = agent.llm_config;
-
-    /** @type {string[]} */
-    let providers = [];
-
-    /** @type {import('$commonTypes').LlmModelSetting[]} */
-    let models = [];
-
-    $: isReasoningModel = models.find(x => x.name === config.model)?.reasoning != null;
+    /** @type {import('$commonTypes').LlmConfig[]} */
+    let llmConfigs = [];
 
     onMount(async () =>{
         await init();
     });
 
     async function init() {
-        providers = await getLlmProviders();
-        providers = ['', ...providers]
-        if (!!config.provider) {
-            models = await getLlmProviderModels(config.provider);
-        }
-        const foundProvider = providers.find(x => x === config.provider);
-        const foundModel = models.find(x => x.name === config.model);
-        config.provider = foundProvider || null;
-        config.model = foundModel?.name || null;
-    }
-
-    
-
-    /** @param {any} e */
-    async function changeProvider(e) {
-        const provider = e.target.value;
-        config.provider = provider || null;
-
-        if (!!!provider) {
-            models = [];
-            config.model = null;
-            config.reasoning_effort_level = null;
-            handleAgentChange();
-            return;
-        }
-
-        config.is_inherit = false;
-        models = await getLlmProviderModels(provider);
-        config.model = models[0]?.name;
-        handleAgentChange();
-    }
-
-    /** @param {any} e */
-    function changeModel(e) {
-        config.is_inherit = false;
-        config.model = e.target.value || null;
-        handleAgentChange();
-    }
-
-    /** @param {any} e */
-    function changeMaxRecursiveDepth(e) {
-        let value = Number(e.target.value) || 0;
+        llmConfigs = await getLlmConfigs();
         
-        if (value < recursiveDepthLowerLimit) {
-            value = recursiveDepthLowerLimit;
-        }
-
-        config.max_recursion_depth = value;
-        handleAgentChange();
-    }
-
-    /** @param {any} e */
-    function changeMaxOutputToken(e) {
-        const value = Number(e.target.value) || 0;
-        config.max_output_tokens = value;
-        handleAgentChange();
-    }
-
-    /** @param {any} e */
-    function changeReasoningEffortLevel(e) {
-        config.reasoning_effort_level = e.target.value || null;
-        handleAgentChange();
-    }
-
-    /** @param {any} e */
-    function validateIntegerInput(e) {
-        const reg = new RegExp(INTEGER_REGEX, 'g');
-        if (e.key !== 'Backspace' && !reg.test(e.key)) {
-            e.preventDefault();
-        }
     }
 </script>
 
 <Card>
     <CardBody>
         <div class="text-center">
-            <h5 class="mt-1 mb-3">LLM Config</h5>
+            <h5 class="mt-1 mb-1">LLM Configurations</h5>
             <img src="images/brands/azure-openai-logo.avif" alt="" height="50" />
-            {#if agent.llm_config?.is_inherit}
-                <i class="bx bx-copy"></i> <span class="text-muted">Inherited</span>    
-            {/if}
-        </div>
-
-        <div class="mb-3 row">
-            <label for="example-large" class="col-md-3 col-form-label">
-                Provider
-            </label>
-            <div class="col-md-9 config-item-container">
-                <Input type="select" value={config.provider} on:change={e => changeProvider(e)}>
-                    {#each providers as option}
-                        <option value={option} selected={option == config.provider}>{option}</option>
-                    {/each}
-                </Input>
-            </div>
-        </div>
-        
-        <div class="mb-3 row">
-            <label for="example-text-input" class="col-md-3 col-form-label">
-                Model
-            </label>
-            <div class="col-md-9">
-                <Input type="select" value={config.model} disabled={models.length === 0} on:change={e => changeModel(e)}>
-                    {#each models as option}
-                        <option value={option.name} selected={option.name == config.model}>{option.name}</option>
-                    {/each}
-                </Input>
-            </div>
-        </div>
-
-        <div class="mb-3 row">
-            <label for="example-text-input" class="col-md-3 col-form-label">
-                Max recursive depth
-            </label>
-            <div class="col-md-9">
-                <Input
-                    style="text-align: center;"
-                    type="number"
-                    min={recursiveDepthLowerLimit}
-                    value={config.max_recursion_depth}
-                    on:keydown={e => validateIntegerInput(e)}
-                    on:change={e => changeMaxRecursiveDepth(e)}
-                />
-            </div>
-        </div>
-
-        <div class="mb-3 row">
-            <label for="example-text-input" class="col-md-3 col-form-label">
-                Max output tokens
-            </label>
-            <div class="col-md-9">
-                <Input
-                    style="text-align: center;"
-                    type="number"
-                    value={config.max_output_tokens}
-                    on:keydown={e => validateIntegerInput(e)}
-                    on:change={e => changeMaxOutputToken(e)}
-                />
-            </div>
         </div>
 
-        {#if isReasoningModel}
-        <div class="mb-3 row">
-            <label for="example-text-input" class="col-md-3 col-form-label">
-                Reasoning effort
-            </label>
-            <div class="col-md-9">
-                <Input type="select" value={config.reasoning_effort_level} on:change={e => changeReasoningEffortLevel(e)}>
-                    {#each reasonLevelOptions as option}
-                        <option value={option.value} selected={option.value == config.reasoning_effort_level}>
-                            {option.label}
-                        </option>
-                    {/each}
-                </Input>
-            </div>
+        <div class="agent-utility-container">
+            <ChatConfig bind:this={chatConfigCmp} {agent} {llmConfigs} {handleAgentChange} />
+            <ImageGenerationConfig bind:this={imageGenerationConfigCmp} {agent} {llmConfigs} {handleAgentChange} />
+            <ImageEditConfig bind:this={imageEditConfigCmp} {agent} {llmConfigs} {handleAgentChange} />
+            <AudioTranscriptionConfig bind:this={audioTranscriptionConfigCmp} {agent} {llmConfigs} {handleAgentChange} />
+            <RealtimeConfig bind:this={realtimeConfigCmp} {agent} {llmConfigs} {handleAgentChange} />
         </div>
-        {/if}
     </CardBody>
 </Card>