diff --git a/src/composables/useDevice.ts b/src/composables/useDevice.ts index 785f61ba..d88e597a 100644 --- a/src/composables/useDevice.ts +++ b/src/composables/useDevice.ts @@ -4,6 +4,7 @@ import { cursorPosition } from '@tauri-apps/api/window' import { INVOKE_KEY, LISTEN_KEY } from '../constants' +import { useMicrophone } from './useMicrophone' import { useModel } from './useModel' import { useTauriListen } from './useTauriListen' @@ -39,6 +40,7 @@ export function useDevice() { const releaseTimers = new Map() const catStore = useCatStore() const { handlePress, handleRelease, handleMouseChange, handleMouseMove } = useModel() + useMicrophone() const startListening = () => { invoke(INVOKE_KEY.START_DEVICE_LISTENING) diff --git a/src/composables/useMicrophone.ts b/src/composables/useMicrophone.ts new file mode 100644 index 00000000..439a99de --- /dev/null +++ b/src/composables/useMicrophone.ts @@ -0,0 +1,229 @@ +import { message } from 'ant-design-vue' +import { onUnmounted, ref, watch } from 'vue' + +import { useCatStore } from '@/stores/cat' +import { useModelStore } from '@/stores/model' +import live2d from '@/utils/live2d' + +export function useMicrophone() { + const catStore = useCatStore() + const modelStore = useModelStore() + + const audioContext = ref(null) + const analyser = ref(null) + const microphone = ref(null) + const mediaStream = ref(null) + const animationFrameId = ref(null) + + const volumeLevel = ref(0) // 当前音量级别 (0-100) + const frequencyData = ref(new Uint8Array(0)) + const timeDomainData = ref(new Uint8Array(0)) + + // 错误处理映射 + const errorMessages: Record = { + NotAllowedError: '麦克风权限被拒绝,请在系统设置中启用', + NotFoundError: '未找到可用的麦克风设备', + NotReadableError: '麦克风设备被其他应用占用', + OverconstrainedError: '无法满足音频约束条件', + SecurityError: '安全限制阻止访问麦克风', + AbortError: '音频设备访问被中止', + } + + // 初始化音频上下文和麦克风 + async function startMicrophone() { + try { + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + throw new Error('当前环境不支持麦克风访问') + } + + // 请求麦克风权限 + mediaStream.value = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + sampleRate: 44100, + }, + }) + + // 创建音频上下文 + audioContext.value = new (window.AudioContext || (window as any).webkitAudioContext)() + analyser.value = audioContext.value.createAnalyser() + + // 配置分析器 + analyser.value.fftSize = 2048 + analyser.value.smoothingTimeConstant = catStore.model.microphoneSmoothing / 100 + + microphone.value = audioContext.value.createMediaStreamSource(mediaStream.value) + microphone.value.connect(analyser.value) + + frequencyData.value = new Uint8Array(analyser.value.frequencyBinCount) + timeDomainData.value = new Uint8Array(analyser.value.fftSize) + + // 确保音频上下文处于运行状态 + if (audioContext.value.state !== 'running') { + await audioContext.value.resume() + } + + // 开始音频分析循环 + startAudioAnalysis() + } catch (error: any) { + const errorMessage = errorMessages[error.name] || `麦克风访问失败: ${error.message}` + message.error(errorMessage) + stopMicrophone() + // 自动禁用麦克风功能 + catStore.model.microphoneEnabled = false + } + } + + function stopMicrophone() { + if (animationFrameId.value) { + cancelAnimationFrame(animationFrameId.value) + animationFrameId.value = null + } + + if (mediaStream.value) { + mediaStream.value.getTracks().forEach(track => track.stop()) + mediaStream.value = null + } + + if (audioContext.value && audioContext.value.state !== 'closed') { + audioContext.value.close() + } + + audioContext.value = null + analyser.value = null + microphone.value = null + volumeLevel.value = 0 + + // 重置 Live2D 嘴部参数 + resetMouthParameters() + } + + function startAudioAnalysis() { + if (!analyser.value || !audioContext.value) return + + const analyseAudio = () => { + if (!analyser.value || !audioContext.value) { + return + } + + // 更新时间域数据(用于音量计算) + analyser.value.getByteTimeDomainData(timeDomainData.value) + + // 计算音量(RMS) + let sum = 0 + for (let i = 0; i < timeDomainData.value.length; i++) { + const value = (timeDomainData.value[i] - 128) / 128 + sum += value * value + } + const rms = Math.sqrt(sum / timeDomainData.value.length) + + // 应用灵敏度调整 + const sensitivity = catStore.model.microphoneSensitivity / 10 + let adjustedVolume = Math.min(rms * sensitivity * 3, 1) // 3倍增益 + adjustedVolume = adjustedVolume ** 0.7 // 非线性映射,更自然 + + // 转换为百分比 (0-100) + volumeLevel.value = adjustedVolume * 100 + + // 应用阈值过滤 + const threshold = catStore.model.microphoneThreshold / 100 + const effectiveVolume = volumeLevel.value / 100 >= threshold ? volumeLevel.value : 0 + + // 更新Live2D参数 + updateLive2DParameters(effectiveVolume) + + // 继续循环(限制在~60fps) + animationFrameId.value = requestAnimationFrame(analyseAudio) + } + + animationFrameId.value = requestAnimationFrame(analyseAudio) + } + + function resetMouthParameters() { + const mouthOpenRange = live2d.getParameterValueRange('ParamMouthOpenY') + if (mouthOpenRange) { + live2d.setParameterValue('ParamMouthOpenY', mouthOpenRange.min) + } + const mouthFormRange = live2d.getParameterValueRange('ParamMouthForm') + if (mouthFormRange) { + live2d.setParameterValue('ParamMouthForm', (mouthFormRange.min + mouthFormRange.max) / 2) + } + } + + function updateLive2DParameters(volume: number) { + if (!modelStore.currentModel) return + + // 映射到嘴部开合参数 ParamMouthOpenY + const mouthOpenRange = live2d.getParameterValueRange('ParamMouthOpenY') + if (mouthOpenRange) { + const { min, max } = mouthOpenRange + // volume是0-100,映射到参数范围 + const mouthOpenValue = min + (volume / 100) * (max - min) + live2d.setParameterValue('ParamMouthOpenY', mouthOpenValue) + } + + // 映射到嘴型参数 ParamMouthForm(基于频率特征) + if (analyser.value && audioContext.value) { + // 获取频率数据 + analyser.value.getByteFrequencyData(frequencyData.value) + + // 寻找主导频率(85-255Hz人声范围) + const sampleRate = audioContext.value.sampleRate + const frequencyResolution = sampleRate / analyser.value.fftSize + const minIndex = Math.floor(85 / frequencyResolution) + const maxIndex = Math.floor(255 / frequencyResolution) + + let maxValue = 0 + let dominantIndex = minIndex + + for (let i = minIndex; i < maxIndex; i++) { + if (frequencyData.value[i] > maxValue) { + maxValue = frequencyData.value[i] + dominantIndex = i + } + } + + const dominantFrequency = dominantIndex * frequencyResolution + + // 将频率映射到嘴型参数 (85-255Hz 映射到 0-1) + const normalizedFreq = Math.max(0, Math.min(1, (dominantFrequency - 85) / (255 - 85))) + + const mouthFormRange = live2d.getParameterValueRange('ParamMouthForm') + if (mouthFormRange) { + const { min, max } = mouthFormRange + const mouthFormValue = min + normalizedFreq * (max - min) + live2d.setParameterValue('ParamMouthForm', mouthFormValue) + } + } + } + + // 监听配置变化 + watch(() => catStore.model.microphoneEnabled, (enabled) => { + if (enabled) { + startMicrophone() + } else { + stopMicrophone() + } + }, { immediate: true }) + + // 监听平滑度变化 + watch(() => catStore.model.microphoneSmoothing, (smoothing) => { + if (analyser.value) { + analyser.value.smoothingTimeConstant = smoothing / 100 + } + }, { immediate: true }) + + // 清理 + onUnmounted(() => { + stopMicrophone() + }) + + return { + startMicrophone, + stopMicrophone, + volumeLevel, + isActive: () => !!mediaStream.value, + } +} diff --git a/src/locales/en-US.json b/src/locales/en-US.json index 2f1e2bf1..3280dbc5 100644 --- a/src/locales/en-US.json +++ b/src/locales/en-US.json @@ -23,7 +23,12 @@ "motionSound": "Motion Sound", "behavior": "Motions and Expressions", "autoReleaseDelay": "Auto Release Delay", - "hideOnHover": "Hide on Hover" + "hideOnHover": "Hide on Hover", + "audioSettings": "Audio Settings", + "microphoneEnabled": "Microphone Input", + "microphoneSensitivity": "Microphone Sensitivity", + "microphoneThreshold": "Activation Threshold", + "microphoneSmoothing": "Smoothing" }, "hints": { "mirrorMode": "When enabled, the model will be mirrored horizontally.", @@ -35,7 +40,11 @@ "keepInScreen": "When enabled, the window automatically stays within the screen boundaries.", "windowSize": "Move mouse to window edge, or hold Shift and right-drag to resize.", "autoReleaseDelay": "On Windows, some system keys cannot capture release events and will auto-release after timeout.", - "hideOnHover": "When enabled, the window hides when mouse hovers over it." + "hideOnHover": "When enabled, the window hides when mouse hovers over it.", + "microphoneEnabled": "When enabled, the model will respond to microphone input volume.", + "microphoneSensitivity": "Adjust the sensitivity of microphone input.", + "microphoneThreshold": "Model actions are triggered when volume reaches this threshold.", + "microphoneSmoothing": "Control the smoothness of mouth movements to avoid jitter." } }, "general": { diff --git a/src/locales/pt-BR.json b/src/locales/pt-BR.json index 1e748e5a..27b16903 100644 --- a/src/locales/pt-BR.json +++ b/src/locales/pt-BR.json @@ -23,7 +23,12 @@ "motionSound": "Som de Ação", "behavior": "Movimentos e Expressões", "autoReleaseDelay": "Atraso de Liberação Automática", - "hideOnHover": "Ocultar ao Passar o Mouse" + "hideOnHover": "Ocultar ao Passar o Mouse", + "audioSettings": "Configurações de Áudio", + "microphoneEnabled": "Entrada de Microfone", + "microphoneSensitivity": "Sensibilidade do Microfone", + "microphoneThreshold": "Limiar de Ativação", + "microphoneSmoothing": "Suavização" }, "hints": { "mirrorMode": "Quando ativado, o modelo será invertido horizontalmente.", @@ -35,7 +40,11 @@ "keepInScreen": "Quando ativado, a janela permanece automaticamente dentro dos limites da tela.", "windowSize": "Mova o mouse para a borda da janela ou segure Shift e arraste com o botão direito para redimensionar.", "autoReleaseDelay": "Devido ao Windows não capturar eventos de liberação de certas teclas de nível do sistema, elas serão automaticamente tratadas como liberadas após um tempo limite.", - "hideOnHover": "Quando ativado, a janela será ocultada quando o mouse passar sobre ela." + "hideOnHover": "Quando ativado, a janela será ocultada quando o mouse passar sobre ela.", + "microphoneEnabled": "Quando ativado, o modelo responderá ao volume de entrada do microfone.", + "microphoneSensitivity": "Ajuste a sensibilidade da entrada do microfone.", + "microphoneThreshold": "Ações do modelo são acionadas quando o volume atinge esse limiar.", + "microphoneSmoothing": "Controle a suavidade dos movimentos da boca para evitar tremores." } }, "general": { diff --git a/src/locales/vi-VN.json b/src/locales/vi-VN.json index b24eb5d4..9fa2c1d1 100644 --- a/src/locales/vi-VN.json +++ b/src/locales/vi-VN.json @@ -23,7 +23,12 @@ "motionSound": "Âm thanh hành động", "behavior": "Hành động và Biểu cảm", "autoReleaseDelay": "Độ trễ tự động nhả phím", - "hideOnHover": "Ẩn khi di chuột" + "hideOnHover": "Ẩn khi di chuột", + "audioSettings": "Cài đặt Âm thanh", + "microphoneEnabled": "Đầu vào Microphone", + "microphoneSensitivity": "Độ nhạy Microphone", + "microphoneThreshold": "Ngưỡng kích hoạt", + "microphoneSmoothing": "Độ mượt" }, "hints": { "mirrorMode": "Bật để lật ngang mô hình.", @@ -35,7 +40,11 @@ "keepInScreen": "Khi bật, cửa sổ sẽ tự động giữ trong ranh giới màn hình.", "windowSize": "Di chuyển chuột đến mép cửa sổ hoặc giữ Shift và kéo chuột phải để thay đổi kích thước.", "autoReleaseDelay": "Do Windows không bắt được sự kiện nhả của một số phím hệ thống, các phím đó sẽ được tự động xem như đã nhả sau khi hết thời gian chờ.", - "hideOnHover": "Khi bật, cửa sổ sẽ ẩn khi chuột di chuyển vào." + "hideOnHover": "Khi bật, cửa sổ sẽ ẩn khi chuột di chuyển vào.", + "microphoneEnabled": "Khi bật, mô hình sẽ phản ứng với âm lượng đầu vào từ microphone.", + "microphoneSensitivity": "Điều chỉnh độ nhạy của đầu vào microphone.", + "microphoneThreshold": "Hành động của mô hình được kích hoạt khi âm lượng đạt ngưỡng này.", + "microphoneSmoothing": "Kiểm soát độ mượt của chuyển động miệng để tránh giật." } }, "general": { diff --git a/src/locales/zh-CN.json b/src/locales/zh-CN.json index 44283080..93c796cf 100644 --- a/src/locales/zh-CN.json +++ b/src/locales/zh-CN.json @@ -23,7 +23,12 @@ "motionSound": "动作音效", "behavior": "动作与表情", "autoReleaseDelay": "按键自动释放延迟", - "hideOnHover": "鼠标移入隐藏" + "hideOnHover": "鼠标移入隐藏", + "audioSettings": "音频设置", + "microphoneEnabled": "麦克风输入", + "microphoneSensitivity": "麦克风灵敏度", + "microphoneThreshold": "触发阈值", + "microphoneSmoothing": "平滑度" }, "hints": { "mirrorMode": "启用后,模型将水平镜像翻转。", @@ -35,7 +40,11 @@ "keepInScreen": "启用后,窗口会自动调整位置,防止超出屏幕边界。", "windowSize": "将鼠标移至窗口边缘,或按住 Shift 并右键拖动,也可以调整窗口大小。", "autoReleaseDelay": "由于 Windows 下部分系统级按键无法捕获释放事件,超时后将自动视为已释放。", - "hideOnHover": "启用后,鼠标悬停在窗口上时,窗口会隐藏。" + "hideOnHover": "启用后,鼠标悬停在窗口上时,窗口会隐藏。", + "microphoneEnabled": "启用后,模型会根据麦克风输入音量做出反应", + "microphoneSensitivity": "调整麦克风输入的灵敏度", + "microphoneThreshold": "音量达到此阈值时触发模型动作", + "microphoneSmoothing": "控制嘴部动作的平滑程度,避免抖动" } }, "general": { diff --git a/src/locales/zh-TW.json b/src/locales/zh-TW.json index 2f4b8e58..c63ad3d6 100644 --- a/src/locales/zh-TW.json +++ b/src/locales/zh-TW.json @@ -23,7 +23,12 @@ "motionSound": "動作音效", "behavior": "動作與表情", "autoReleaseDelay": "按鍵自動釋放延遲", - "hideOnHover": "滑鼠游標移入隱藏" + "hideOnHover": "滑鼠游標移入隱藏", + "audioSettings": "音訊設定", + "microphoneEnabled": "麥克風輸入", + "microphoneSensitivity": "麥克風靈敏度", + "microphoneThreshold": "觸發閾值", + "microphoneSmoothing": "平滑度" }, "hints": { "mirrorMode": "啟用後,模型將水平鏡像翻轉。", @@ -35,7 +40,11 @@ "keepInScreen": "啟用後,視窗會自動調整位置,防止超出螢幕邊界。", "windowSize": "將滑鼠游標移至視窗邊緣,或按住 Shift 並右鍵拖曳,也可以調整視窗大小。", "autoReleaseDelay": "由於 Windows 下部份系統級按鍵無法擷取釋放事件,超時後將自動視為已釋放。", - "hideOnHover": "啟用後,滑鼠游標懸停在視窗上時,視窗會隱藏。" + "hideOnHover": "啟用後,滑鼠游標懸停在視窗上時,視窗會隱藏。", + "microphoneEnabled": "啟用後,模型會根據麥克風輸入音量做出反應", + "microphoneSensitivity": "調整麥克風輸入的靈敏度", + "microphoneThreshold": "音量達到此閾值時觸發模型動作", + "microphoneSmoothing": "控制嘴部動作的平滑程度,避免抖動" } }, "general": { diff --git a/src/pages/preference/components/cat/index.vue b/src/pages/preference/components/cat/index.vue index 211f6761..f5f281df 100644 --- a/src/pages/preference/components/cat/index.vue +++ b/src/pages/preference/components/cat/index.vue @@ -116,4 +116,57 @@ const catStore = useCatStore() /> + + + + + + + + diff --git a/src/stores/cat.ts b/src/stores/cat.ts index c7865f0e..13374c4a 100644 --- a/src/stores/cat.ts +++ b/src/stores/cat.ts @@ -8,6 +8,10 @@ export interface CatStore { motionSound: boolean behavior: boolean autoReleaseDelay: number + microphoneEnabled: boolean + microphoneSensitivity: number + microphoneThreshold: number + microphoneSmoothing: number } window: { visible: boolean @@ -51,6 +55,10 @@ export const useCatStore = defineStore('cat', () => { motionSound: true, behavior: true, autoReleaseDelay: 3, + microphoneEnabled: false, + microphoneSensitivity: 50, + microphoneThreshold: 30, + microphoneSmoothing: 70, }) const window = reactive({