Merge pull request #55 from pragmatrix/azure-transcribe-multi-language

Support multiple language detection in azure-transcribe.

Author: pragmatrix

core/src/conversation.rs

@@ -227,8 +227,12 @@ impl ConversationOutput {
         self.post(Output::ClearAudio)
     }
 
-    pub fn text(&self, is_final: bool, text: String) -> Result<()> {
-        self.post(Output::Text { is_final, text })
+    pub fn text(&self, is_final: bool, text: String, language: Option<String>) -> Result<()> {
+        self.post(Output::Text {
+            is_final,
+            text,
+            language,
+        })
     }
 
     pub fn request_completed(&self, request_id: Option<RequestId>) -> Result<()> {
@@ -318,6 +322,7 @@ pub enum Output {
     Text {
         is_final: bool,
         text: String,
+        language: Option<String>,
     },
     RequestCompleted {
         request_id: Option<RequestId>,

examples/azure-transcribe.rs

@@ -15,7 +15,7 @@ use context_switch_core::{
     service::Service,
 };
 
-const LANGUAGE: &str = "de-DE";
+const LANGUAGE: &str = "de-DE,en-US";
 
 #[tokio::main]
 async fn main() -> Result<()> {

examples/azure-translate.rs

@@ -178,8 +178,12 @@ async fn setup_audio_playback(
                         break;
                     }
                 }
-                Output::Text { is_final, text } => {
-                    println!("Text: {text}, final: {is_final}")
+                Output::Text {
+                    is_final,
+                    text,
+                    language,
+                } => {
+                    println!("Text: {text}, final: {is_final}, language: {language:?}")
                 }
                 Output::RequestCompleted { .. } => {}
                 Output::ClearAudio => {

services/aristech/src/transcribe.rs

@@ -152,7 +152,7 @@ impl Service for AristechTranscribe {
 
                 // Instead of processing all alternatives, just take the first one
                 if let Some(alternative) = chunk.alternatives.into_iter().next() {
-                    output.text(is_final, alternative.text)?;
+                    output.text(is_final, alternative.text, None)?;
                 }
             }
         }

services/azure/src/transcribe.rs

@@ -9,7 +9,7 @@ use tracing::{error, info};
 use crate::Host;
 use context_switch_core::{
     BillingRecord, Service,
-    conversation::{BillingSchedule, Conversation, Input},
+    conversation::{BillingSchedule, Conversation, ConversationOutput, Input},
     speech_gate::make_speech_gate_processor_soft_rms,
 };
 
@@ -46,13 +46,35 @@ impl Service for AzureTranscribe {
             }
         };
 
+        let languages = params
+            .language
+            .split(',')
+            .map(str::trim)
+            .filter(|x| !x.is_empty())
+            .map(str::to_owned)
+            .collect::<Vec<_>>();
+
+        if languages.is_empty() {
+            bail!("language must contain at least one locale code");
+        }
+        let include_detected_language = languages.len() > 1;
+
         let config = recognizer::Config::default()
             // Disable profanity filter.
-            .set_profanity(recognizer::Profanity::Raw)
-            // short-circuit language filter.
-            // TODO: may actually use the filter to check for supported languages?
-            .set_language(recognizer::Language::Custom(params.language))
-            .set_output_format(recognizer::OutputFormat::Detailed);
+            .set_profanity(recognizer::Profanity::Raw);
+
+        let config = if languages.len() == 1 {
+            config.set_language(recognizer::Language::Custom(languages[0].clone()))
+        } else {
+            config.set_detect_languages(
+                languages
+                    .into_iter()
+                    .map(recognizer::Language::Custom)
+                    .collect(),
+                recognizer::LanguageDetectMode::Continuous,
+            )
+        }
+        .set_output_format(recognizer::OutputFormat::Detailed);
 
         let client = recognizer::Client::connect(host.auth.clone(), config).await?;
 
@@ -108,13 +130,36 @@ impl Service for AzureTranscribe {
                 | Event::StartDetected(_, _)
                 | Event::EndDetected(_, _) => {}
                 Event::Recognizing(_, recognized, _, _, _) => {
-                    output.text(false, recognized.text)?
+                    output_recognized_text(&output, recognized, false, include_detected_language)?
+                }
+                Event::Recognized(_, recognized, _, _, _) => {
+                    output_recognized_text(&output, recognized, true, include_detected_language)?
                 }
-                Event::Recognized(_, recognized, _, _, _) => output.text(true, recognized.text)?,
                 Event::UnMatch(_, _, _, _) => {}
             }
         }
 
         Ok(())
     }
 }
+
+fn output_recognized_text(
+    output: &ConversationOutput,
+    recognized: recognizer::Recognized,
+    is_final: bool,
+    include_detected_language: bool,
+) -> Result<()> {
+    let recognizer::Recognized {
+        text,
+        primary_language,
+        ..
+    } = recognized;
+
+    let language = if include_detected_language {
+        primary_language.map(|x| x.language.to_string())
+    } else {
+        None
+    };
+
+    output.text(is_final, text, language)
+}

services/azure/src/translate.rs

@@ -128,12 +128,12 @@ impl Service for AzureTranslate {
                 Event::EndDetected(_, _) => {}
                 Event::Translating(_, text, _, _, _) => {
                     if output_modalities.text {
-                        output.text(false, text)?;
+                        output.text(false, text, None)?;
                     }
                 }
                 Event::Translated(_, text, _, _, _) => {
                     if output_modalities.interim_text {
-                        output.text(true, text)?;
+                        output.text(true, text, None)?;
                     }
                 }
                 Event::TranslationSynthesis(_, samples) => {