feat: embeding silero_vad for whisper

Author: L-jasmine

Cargo.lock

@@ -51,6 +51,7 @@ rmcp = { version = "0.1.5", features = [
     "transport-streamable-http-client",
     "reqwest",
 ], default-features = false, git = "https://github.com/modelcontextprotocol/rust-sdk", rev = "b9d7d61" } # branch = "main"
+
 base64 = "0.22.1"
 reqwest-websocket = "0.5.0"
 futures-util = "0.3.31"
@@ -59,3 +60,7 @@ tower = { version = "0.5.2", features = ["util"] }
 tower-http = { version = "0.6.1", features = ["fs", "trace"] }
 
 chrono = "0.4.41"
+
+# vad
+silero_vad_burn = "0.1.1"
+burn = { version = "0.20", features = ["ndarray"] }

Cargo.toml

@@ -473,7 +473,7 @@ pub async fn llm_stable<'p, I: IntoIterator<Item = C>, C: AsRef<llm::Content>>(
         serde_json::to_string_pretty(&serde_json::json!(
             {
                 "stream": true,
-                "messages": messages,
+                "last_message": messages.last(),
                 "model": model.to_string(),
                 "tools": tool_name,
                 "extra": extra,

src/ai/mod.rs

@@ -1,6 +1,6 @@
 use futures_util::{
-    stream::{SplitSink, SplitStream},
     SinkExt, StreamExt,
+    stream::{SplitSink, SplitStream},
 };
 use reqwest::multipart::Part;
 use reqwest_websocket::{RequestBuilderExt, WebSocket};
@@ -101,3 +101,109 @@ impl VadRealtimeRx {
         }
     }
 }
+
+pub type VadParams = crate::config::SileroVadconfig;
+
+#[derive(Clone)]
+pub struct SileroVADFactory {
+    device: burn::backend::ndarray::NdArrayDevice,
+    params: VadParams,
+}
+
+impl SileroVADFactory {
+    pub fn new(params: VadParams) -> anyhow::Result<Self> {
+        let device = burn::backend::ndarray::NdArrayDevice::default();
+
+        Ok(SileroVADFactory { device, params })
+    }
+
+    pub fn create_session(&self) -> anyhow::Result<VadSession> {
+        let vad = Box::new(silero_vad_burn::SileroVAD6Model::new(&self.device)?);
+        VadSession::new(&self.params, vad, self.device.clone())
+    }
+}
+
+pub struct VadSession {
+    vad: Box<silero_vad_burn::SileroVAD6Model<burn::backend::NdArray>>,
+    state: Option<silero_vad_burn::PredictState<burn::backend::NdArray>>,
+    device: burn::backend::ndarray::NdArrayDevice,
+
+    in_speech: bool,
+
+    threshold: f32,
+    neg_threshold: f32,
+
+    silence_chunk_count: usize,
+    max_silence_chunks: usize,
+}
+
+impl VadSession {
+    const SAMPLE_RATE: usize = 16000;
+
+    pub fn new(
+        params: &VadParams,
+        vad: Box<silero_vad_burn::SileroVAD6Model<burn::backend::NdArray>>,
+        device: burn::backend::ndarray::NdArrayDevice,
+    ) -> anyhow::Result<Self> {
+        let state = Some(silero_vad_burn::PredictState::default(&device));
+
+        let neg_threshold = params
+            .neg_threshold
+            .unwrap_or_else(|| params.threshold - 0.15)
+            .max(0.05);
+
+        let threshold = params.threshold.min(0.95);
+        let max_silence_chunks = params.max_silence_duration_ms * (Self::SAMPLE_RATE / 1000)
+            / silero_vad_burn::CHUNK_SIZE;
+
+        Ok(VadSession {
+            vad,
+            state,
+            device,
+
+            in_speech: false,
+            threshold,
+            neg_threshold,
+
+            silence_chunk_count: 0,
+            max_silence_chunks,
+        })
+    }
+
+    pub fn reset_state(&mut self) {
+        self.state = Some(silero_vad_burn::PredictState::default(&self.device));
+        self.in_speech = false;
+        self.silence_chunk_count = 0;
+    }
+
+    pub fn detect(&mut self, audio16k_chunk_512: &[f32]) -> anyhow::Result<bool> {
+        debug_assert!(
+            audio16k_chunk_512.len() <= 512,
+            "audio16k_chunk_512 length must be less than 512",
+        );
+
+        let audio_tensor =
+            burn::Tensor::<_, 1>::from_floats(audio16k_chunk_512, &self.device).unsqueeze();
+        let (state, prob) = self.vad.predict(self.state.take().unwrap(), audio_tensor)?;
+        self.state = Some(state);
+
+        let prob: Vec<f32> = prob.to_data().to_vec()?;
+
+        if prob[0] > self.threshold {
+            self.in_speech = true;
+            self.silence_chunk_count = 0;
+        } else if prob[0] < self.neg_threshold {
+            self.silence_chunk_count += 1;
+            if self.silence_chunk_count >= self.max_silence_chunks {
+                self.in_speech = false;
+            }
+        } else {
+        }
+
+        Ok(self.in_speech)
+    }
+
+    pub const fn vad_chunk_size() -> usize {
+        silero_vad_burn::CHUNK_SIZE
+    }
+}

src/ai/vad.rs

@@ -242,6 +242,50 @@ pub enum TTSConfig {
     Elevenlabs(ElevenlabsTTS),
 }
 
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct SileroVadconfig {
+    pub threshold: f32,
+    pub neg_threshold: Option<f32>,
+
+    pub min_speech_duration_ms: usize,
+    pub max_silence_duration_ms: usize,
+    pub audio_cache_ms: usize,
+}
+
+impl SileroVadconfig {
+    pub fn default_threshold() -> f32 {
+        0.5
+    }
+
+    pub fn default_neg_threshold() -> Option<f32> {
+        None
+    }
+
+    pub fn default_min_speech_duration_ms() -> usize {
+        150
+    }
+
+    pub fn default_max_silence_duration_ms() -> usize {
+        400
+    }
+
+    pub fn default_audio_cache_ms() -> usize {
+        1000
+    }
+}
+
+impl Default for SileroVadconfig {
+    fn default() -> Self {
+        SileroVadconfig {
+            threshold: Self::default_threshold(),
+            neg_threshold: Self::default_neg_threshold(),
+            min_speech_duration_ms: Self::default_min_speech_duration_ms(),
+            max_silence_duration_ms: Self::default_max_silence_duration_ms(),
+            audio_cache_ms: Self::default_audio_cache_ms(),
+        }
+    }
+}
+
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct WhisperASRConfig {
     pub url: String,
@@ -253,8 +297,14 @@ pub struct WhisperASRConfig {
     pub model: String,
     #[serde(default)]
     pub prompt: String,
+
+    #[serde(default)]
+    pub vad: SileroVadconfig,
+
+    #[deprecated]
     #[serde(default)]
     pub vad_url: Option<String>,
+    #[deprecated]
     #[serde(default)]
     pub vad_realtime_url: Option<String>,
 }

src/config.rs

@@ -186,11 +186,10 @@ async fn routes(
     router.route(
         "/version",
         get(|| async {
-            serde_json::to_string_pretty(&serde_json::json!(
+            axum::response::Json(serde_json::json!(
             {
                 "version": env!("CARGO_PKG_VERSION"),
             }))
-            .unwrap()
         }),
     )
 }

src/main.rs

@@ -16,6 +16,8 @@ pub enum ServerEvent {
     StartVideo,
     EndVideo,
     EndResponse,
+
+    EndVad,
 }
 
 #[test]

src/protocol.rs

@@ -21,6 +21,8 @@ pub struct ConnectQueryParams {
     opus: bool,
     #[serde(default)]
     vowel: bool,
+    #[serde(default)]
+    server_vad: bool,
 }
 
 pub async fn v2_mixed_handler(
@@ -43,6 +45,7 @@ pub async fn v2_mixed_handler(
                 reconnect: params.reconnect,
                 opus: params.opus,
                 vowel: params.vowel,
+                server_vad: params.server_vad,
             }),
         )
         .await