Skip to content

feat: Intent STT TTS node support model reference #4948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhanweizhang7 merged 1 commit into from
Mar 25, 2026
Merged

feat: Intent STT TTS node support model reference #4948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions apps/application/flow/step_node/intent_node/i_intent_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ class IntentBranchSerializer(serializers.Serializer):


class IntentNodeSerializer(serializers.Serializer):
model_id = serializers.CharField(required=True, label=_("Model id"))
model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id"))
model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type"))
model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True,
label=_("Reference Field"))
content_list = serializers.ListField(required=True, label=_("Text content"))
dialogue_number = serializers.IntegerField(required=True, label=
_("Number of multi-round conversations"))
Expand Down Expand Up @@ -52,5 +55,5 @@ def _run(self):
user_input=str(question))

def execute(self, model_id, dialogue_number, history_chat_record, user_input, branch,
model_params_setting=None, **kwargs) -> NodeResult:
model_params_setting=None, model_id_type=None, model_id_reference=None, **kwargs) -> NodeResult:
pass
13 changes: 11 additions & 2 deletions apps/application/flow/step_node/intent_node/impl/base_intent_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,19 @@ def save_context(self, details, workflow_manage):
self.context['category'] = details.get('category')

def execute(self, model_id, dialogue_number, history_chat_record, user_input, branch,
model_params_setting=None, **kwargs) -> NodeResult:
model_params_setting=None, model_id_type=None, model_id_reference=None, **kwargs) -> NodeResult:
# 处理引用类型
if model_id_type == 'reference' and model_id_reference:
reference_data = self.workflow_manage.get_reference_field(
model_id_reference[0],
model_id_reference[1:],
)
if reference_data and isinstance(reference_data, dict):
model_id = reference_data.get('model_id', model_id)
model_params_setting = reference_data.get('model_params_setting')

# 设置默认模型参数
if model_params_setting is None:
if model_params_setting is None and model_id:
model_params_setting = get_default_model_params_setting(model_id)

# 获取模型实例
Expand Down
8 changes: 5 additions & 3 deletions apps/application/flow/step_node/speech_to_text_step_node/i_speech_to_text_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@


class SpeechToTextNodeSerializer(serializers.Serializer):
stt_model_id = serializers.CharField(required=True, label=_("Model id"))

stt_model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id"))
stt_model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type"))
stt_model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True,
label=_("Reference Field"))
is_result = serializers.BooleanField(required=False,
label=_('Whether to return content'))

Expand Down Expand Up @@ -40,6 +42,6 @@ def _run(self):
return self.execute(audio=res, **self.node_params_serializer.data, **self.flow_params_serializer.data)

def execute(self, stt_model_id,
audio, model_params_setting=None,
audio, model_params_setting=None, stt_model_id_type=None, stt_model_id_reference=None,
**kwargs) -> NodeResult:
pass
13 changes: 12 additions & 1 deletion apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,18 @@ def save_context(self, details, workflow_manage):
self.answer_text = details.get('answer')
self.context['exception_message'] = details.get('err_message')

def execute(self, stt_model_id, audio, model_params_setting=None, **kwargs) -> NodeResult:
def execute(self, stt_model_id, audio, model_params_setting=None, stt_model_id_type=None, stt_model_id_reference=None,**kwargs) -> NodeResult:

# 处理引用类型
if stt_model_id_type == 'reference' and stt_model_id_reference:
reference_data = self.workflow_manage.get_reference_field(
stt_model_id_reference[0],
stt_model_id_reference[1:],
)
if reference_data and isinstance(reference_data, dict):
stt_model_id = reference_data.get('stt_model_id', reference_data.get('model_id', stt_model_id))
model_params_setting = reference_data.get('model_params_setting')

workspace_id = self.workflow_manage.get_body().get('workspace_id')
stt_model = get_model_instance_by_model_workspace_id(stt_model_id, workspace_id, **(model_params_setting or {}))
audio_list = audio
Expand Down
8 changes: 5 additions & 3 deletions apps/application/flow/step_node/text_to_speech_step_node/i_text_to_speech_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@


class TextToSpeechNodeSerializer(serializers.Serializer):
tts_model_id = serializers.CharField(required=True, label=_("Model id"))

tts_model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id"))
tts_model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type"))
tts_model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True,
label=_("Reference Field"))
is_result = serializers.BooleanField(required=False,
label=_('Whether to return content'))

Expand All @@ -35,6 +37,6 @@ def _run(self):
return self.execute(content=content, **self.node_params_serializer.data, **self.flow_params_serializer.data)

def execute(self, tts_model_id,
content, model_params_setting=None,
content, model_params_setting=None, tts_model_id_type=None, tts_model_id_reference=None,
**kwargs) -> NodeResult:
pass
11 changes: 10 additions & 1 deletion apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,17 @@ def save_context(self, details, workflow_manage):
self.answer_text = details.get('answer')

def execute(self, tts_model_id,
content, model_params_setting=None,
content, model_params_setting=None, tts_model_id_type=None, tts_model_id_reference=None,
max_length=1024, **kwargs) -> NodeResult:
# 处理引用类型
if tts_model_id_type == 'reference' and tts_model_id_reference:
reference_data = self.workflow_manage.get_reference_field(
tts_model_id_reference[0],
tts_model_id_reference[1:],
)
if reference_data and isinstance(reference_data, dict):
tts_model_id = reference_data.get('tts_model_id', reference_data.get('model_id', tts_model_id))
model_params_setting = reference_data.get('model_params_setting')
# 分割文本为合理片段
content = _remove_empty_lines(content)
content_chunks = [content[i:i + max_length]
Expand Down
6 changes: 3 additions & 3 deletions ui/src/workflow/nodes/ai-chat-node/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,12 @@
<div class="ml-8">
<el-button
:disabled="!chat_data.model_id"
type="primary"
link
@click="openAIParamSettingDialog(chat_data.model_id)"
@refreshForm="refreshParam"
>
<AppIcon iconName="app-setting"></AppIcon>
<el-icon>
<Operation />
</el-icon>
</el-button>
</div>
</div>
Expand Down
69 changes: 51 additions & 18 deletions ui/src/workflow/nodes/intent-classify-node/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,68 @@
>
<el-form-item
:label="$t('views.application.form.aiModel.label')"
prop="model_id"
:prop="form_data.model_id_type === 'reference' ? 'model_id_reference' : 'model_id'"
:rules="{
required: true,
message: $t('views.application.form.aiModel.placeholder'),
message:
form_data.model_id_type === 'reference'
? $t('workflow.variable.placeholder')
: $t('views.application.form.aiModel.placeholder'),
trigger: 'change',
}"
>
<template #label>
<div class="flex-between">
<div class="flex-between w-full">
<div>
<span
>{{ $t('views.application.form.aiModel.label')
}}<span class="color-danger">*</span></span
>
</div>
<el-select
v-model="form_data.model_id_type"
:teleported="false"
size="small"
style="width: 85px"
@change="form_data.model_id_reference = []"
>
<el-option :label="$t('workflow.variable.Referencing')" value="reference" />
<el-option :label="$t('common.custom')" value="custom" />
</el-select>
</div>
</template>
<div class="flex-between w-full" v-if="form_data.model_id_type !== 'reference'">
<ModelSelect
@change="model_change"
@wheel="wheel"
:teleported="false"
v-model="form_data.model_id"
:placeholder="$t('views.application.form.aiModel.placeholder')"
:options="modelOptions"
@submitModel="getSelectModel"
showFooter
:model-type="'LLM'"
></ModelSelect>
<div class="ml-8">
<el-button
type="primary"
link
:disabled="!form_data.model_id"
@click="openAIParamSettingDialog(form_data.model_id)"
@refreshForm="refreshParam"
>
<AppIcon iconName="app-setting"></AppIcon>
<el-icon>
<Operation />
</el-icon>
</el-button>
</div>
</template>
<ModelSelect
@change="model_change"
@wheel="wheel"
:teleported="false"
v-model="form_data.model_id"
:placeholder="$t('views.application.form.aiModel.placeholder')"
:options="modelOptions"
@submitModel="getSelectModel"
showFooter
:model-type="'LLM'"
></ModelSelect>
</div>
<NodeCascader
v-else
ref="nodeCascaderRef"
:nodeModel="nodeModel"
class="w-full"
:placeholder="$t('workflow.variable.placeholder')"
v-model="form_data.model_id_reference"
/>
</el-form-item>
<el-form-item
prop="content_list"
Expand Down Expand Up @@ -281,6 +306,8 @@ const model_change = (model_id?: string) => {

const form = {
model_id: '',
model_id_type: 'custom',
model_id_reference: [],
branch: [
{
id: randomId(),
Expand Down Expand Up @@ -309,6 +336,12 @@ const openAIParamSettingDialog = (modelId: string) => {
const form_data = computed({
get: () => {
if (props.nodeModel.properties.node_data) {
if (!props.nodeModel.properties.node_data.model_id_type) {
set(props.nodeModel.properties.node_data, 'model_id_type', 'custom')
}
if (!props.nodeModel.properties.node_data.model_id_reference) {
set(props.nodeModel.properties.node_data, 'model_id_reference', [])
}
return props.nodeModel.properties.node_data
} else {
set(props.nodeModel.properties, 'node_data', form)
Copy link
Copy Markdown
Contributor Author

@shaohuzhang1 shaohuzhang1 Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here are some potential issues and suggestions from reviewing the given code:

Potential Issues

  1. Variable Scope Issue:

    • The variable modelChange is declared within an inner function but not defined outside of it. This might indicate that there's an issue with its scope, possibly resulting in undefined behavior.

  2. Missing Function Definitions:

    • getSelectModel, resetModalSettings, and other functions used within Vue templates (v-on directives) need to be properly defined elsewhere in the component or passed down by parent components.

  3. Conditional Rendering Logic:

    • There seems to be redundancy between conditional rendering based on form_data.model_id_type. Consider simplifying the logic where possible.

  4. Component Interaction:

    • The <NodeCascader> component references properties like node_model which aren't defined in the provided snippet. Ensure this component has access to necessary data.

  5. Translation Strings:

    • If $t is called directly within templates without a context (like <lang:.../>) ensure translations are correctly loaded into the application.

Optimization Suggestions

  1. Reduce Repetitive Code:

    • Extract out repetitive code blocks into reusable utility functions to reduce redundancy and improve readability.

  2. Use Vuex Actions for Mutations:

    • For state management, consider using Vuex actions instead of direct mutation logic if you don’t already use one. This can help keep mutations clear and maintainable.

  3. Lazy Loading Components:

    • If <ModelSelect> and <NodeCascader> are large or complex components, consider lazy loading them only when needed during runtime to improve initial app performance.

  4. Optimize Template Structure:

    • Ensure template structure remains clean and optimized. Avoid deeply nested elements to avoid excessive DOM parsing time and memory usage.

  5. Avoid Inline Event Handlers：

    • Where possible, move event handlers (especially those triggered by user input) to methods outside of template logic to enhance reusability and maintainability.

Suggested Improvements

// Example of how the improved version might look
<template>
  <el-form ...>
    <!-- Conditionally render ModelSelect component -->
    <ModelSelect v-if="form_data.model_id_type !== 'reference'" />

    <!-- Conditional render NodeCascader component -->
    <NodeCascader v-else ref="nodeCascaderRef" />

    <!-- Common Form Items -->
    ...
  </el-form>
</template>

<script setup lang="ts">
import { reactive, computed } from 'vue';
import ModelSelect from './ModelSelect.vue';
import NodeCascader from './NodeCascader.vue';

const form_data = reactive({
  // Initialize with appropriate defaults here
});

// Computed property handling model change event
const handleModelChange = (newModelID: string): void => {
  form_data.model_id = newModelID;
};

watch(() => props.nodeModel.properties.node_data, (newValue) => {
  if (!newValue.model_id_type) {
    newValue.model_id_type = 'custom';
  }
  
  if (!newValue.model_id_reference) {
    setValue(newValue, 'model_id_reference', []);
  }

  Object.assign(form_data, newValue);
}, {
  deep: true,
});
</script>

This suggested approach reduces redundancy, optimizes component interactions, enhances scalability, and improves overall code quality through proper separation of concerns within Vue instances via the Composition API pattern introduced in Vue 3.

Expand Down
71 changes: 51 additions & 20 deletions ui/src/workflow/nodes/speech-to-text-node/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@
>
<el-form-item
:label="$t('workflow.nodes.speechToTextNode.stt_model.label')"
prop="stt_model_id"
:prop="
form_data.stt_model_id_type === 'reference' ? 'stt_model_id_reference' : 'stt_model_id'
"
:rules="{
required: true,
message: $t('views.application.form.voiceInput.placeholder'),
message:
form_data.stt_model_id_type === 'reference'
? $t('workflow.variable.placeholder')
: $t('views.application.form.voiceInput.placeholder'),
trigger: 'change',
}"
>
Expand All @@ -28,27 +33,45 @@
}}<span class="color-danger">*</span></span
>
</div>
<el-button
type="primary"
link
@click="openSTTParamSettingDialog"
:disabled="!form_data.stt_model_id"
class="mr-4"
<el-select
v-model="form_data.stt_model_id_type"
:teleported="false"
size="small"
style="width: 85px"
@change="form_data.stt_model_id_reference = []"
>
<AppIcon iconName="app-setting"></AppIcon>
</el-button>
<el-option :label="$t('workflow.variable.Referencing')" value="reference" />
<el-option :label="$t('common.custom')" value="custom" />
</el-select>
</div>
</template>
<ModelSelect
@wheel="wheel"
:teleported="false"
@change="sttModelChange"
v-model="form_data.stt_model_id"
:placeholder="$t('views.application.form.voiceInput.placeholder')"
:options="modelOptions"
showFooter
:model-type="'STT'"
></ModelSelect>
<div class="flex-between w-full" v-if="form_data.stt_model_id_type !== 'reference'">
<ModelSelect
@wheel="wheel"
:teleported="false"
@change="sttModelChange"
v-model="form_data.stt_model_id"
:placeholder="$t('views.application.form.voiceInput.placeholder')"
:options="modelOptions"
showFooter
:model-type="'STT'"
></ModelSelect>
<div class="ml-8">
<el-button @click="openSTTParamSettingDialog" :disabled="!form_data.stt_model_id">
<el-icon>
<Operation />
</el-icon>
</el-button>
</div>
</div>
<NodeCascader
v-else
ref="nodeCascaderRef"
:nodeModel="nodeModel"
class="w-full"
:placeholder="$t('workflow.variable.placeholder')"
v-model="form_data.stt_model_id_reference"
/>
</el-form-item>
<el-form-item
:label="$t('workflow.nodes.speechToTextNode.audio.label')"
Expand Down Expand Up @@ -168,6 +191,8 @@ const wheel = (e: any) => {

const form = {
stt_model_id: '',
stt_model_id_type: 'custom',
stt_model_id_reference: [],
is_result: true,
audio_list: [],
model_params_setting: {},
Expand All @@ -176,6 +201,12 @@ const form = {
const form_data = computed({
get: () => {
if (props.nodeModel.properties.node_data) {
if (!props.nodeModel.properties.node_data.stt_model_id_type) {
set(props.nodeModel.properties.node_data, 'stt_model_id_type', 'custom')
}
if (!props.nodeModel.properties.node_data.stt_model_id_reference) {
set(props.nodeModel.properties.node_data, 'stt_model_id_reference', [])
}
return props.nodeModel.properties.node_data
} else {
set(props.nodeModel.properties, 'node_data', form)
Copy link
Copy Markdown
Contributor Author

@shaohuzhang1 shaohuzhang1 Mar 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The provided code snippet contains several improvements and corrections:

  1. Type Handling: The stt_model_id property now checks its type ('reference') to decide which prop to use ('stt_model_id_reference' or 'stt_model_id'). This enhances flexibility based on user selections.

  2. Optimized Button Placement: A horizontal flex container has been added around the ModelSelect component, making it look cleaner and responsive.

  3. Conditional Rendering: Only shows the NodeCascader when stt_model_id_type is 'reference', improving usability since not all users might need this feature.

  4. Default Property Values: In the createNodePropsData() function, default values are set for properties if they are missing from the node's data, preventing runtime errors.

  5. Placeholder Text Consistency: Added more specific placeholders where necessary, such as $t('common.custom') in the dropdown label.

These changes make the code more robust, intuitive, and maintainable while addressing potential bugs in input validation and layout adjustments.

Note: Make sure translations ($t() calls) are correctly configured and available for the specified languages ('workflow.variable.label', etc.).

Expand Down
Loading
Loading