Audio: MFCC: Add Voice Activity Detection based on Mel spectrum

This patch adds a new mfcc_vad module. It operates  on the Mel
log spectrum values produced by the MFCC component. The VAD is
very simple and not very selective for voice vs. other signals.
But the continuously updated background noise estimate prevents
stationary noises from triggering the VAD.

The algorithm tracks a per-bin noise floor (instant-down, slow-rise)
and computes a A-weighted energy delta. The used weight emphasizes
speech frequencies. Speech is declared when the delta exceeds a
threshold (0.35 in Q9.23) with a 20-frame hangover to prevent rapid
toggling.

The VAD flag is inserted into the output stream as the first value
after the magic header word in all format paths (S16, S24, S32).

A new Kconfig option CONFIG_COMP_MFCC_VAD (depends on COMP_MFCC,
default y) gates compilation of the VAD code and the stream format
change.

The README.txt file is updated to show help how to run the
example Python script sof_mel_to_text_live_dsp_vad.py. It uses
the MFCC Mel spectrum data and VAD flags stream as audio features
for Whisper speech to text model. The formatting is changed to md.

Signed-off-by: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>

Author: singalsu

src/arch/host/configs/library_defconfig

@@ -11,6 +11,7 @@ CONFIG_COMP_IIR=y
 CONFIG_COMP_IGO_NR=y
 CONFIG_COMP_LEVEL_MULTIPLIER=y
 CONFIG_COMP_MFCC=y
+CONFIG_COMP_MFCC_VAD=y
 CONFIG_COMP_MODULE_ADAPTER=y
 CONFIG_COMP_MULTIBAND_DRC=y
 CONFIG_COMP_MUX=y

src/audio/mfcc/CMakeLists.txt

@@ -5,4 +5,7 @@ if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
   add_dependencies(app mfcc)
 else()
   add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c)
+  if(CONFIG_COMP_MFCC_VAD)
+    add_local_sources(sof mfcc_vad.c)
+  endif()
 endif()

src/audio/mfcc/Kconfig

@@ -24,3 +24,14 @@ config COMP_MFCC
 	  The characteristic of the audio features are defined in the binary
 	  control blob. Directory tools/tune/mfcc contains a tool to create
 	  the configurations.
+
+config COMP_MFCC_VAD
+	bool "MFCC Voice Activity Detection"
+	depends on COMP_MFCC
+	default y
+	help
+	  This option enables a Voice Activity Detector (VAD) that operates
+	  on the Mel spectrum values produced by the MFCC component. The VAD
+	  flag is inserted into the output stream as the first int32_t value
+	  after the magic header word. The VAD tracks a per-bin noise floor
+	  and detects speech using a weighted energy delta with hangover.

src/audio/mfcc/mfcc_common.c

@@ -21,6 +21,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef CONFIG_COMP_MFCC_VAD
+#include <sof/audio/mfcc/mfcc_vad.h>
+#endif
+
 LOG_MODULE_REGISTER(mfcc_common, CONFIG_SOF_LOG_LEVEL);
 
 /*
@@ -144,6 +148,10 @@ static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *
 					sat_int32(Q_MULTSR_32X32(s, config->mel_scale, 23, 12, 23));
 			}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+			/* Run VAD on the mel log spectrum before further processing */
+			state->vad_flag = mfcc_vad_update(&cd->vad, state->mel_log_32);
+#endif
 			/* Store Q9.7 version in mel_spectra for s16 output mode */
 			for (j = 0; j < state->dct.num_in; j++)
 				state->mel_spectra->data[j] =
@@ -282,10 +290,14 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
 
 	/* If new output produced, set up pointer into scratch data and mark magic pending */
 	if (num_ceps > 0) {
-		if (state->mel_only)
+		if (state->mel_only) {
 			state->out_data_ptr = state->mel_spectra->data;
-		else
+#ifdef CONFIG_COMP_MFCC_VAD
+			state->vad_pending = true;
+#endif
+		} else {
 			state->out_data_ptr = state->cepstral_coef->data;
+		}
 
 		state->out_remain = num_ceps;
 		state->magic_pending = true;
@@ -301,6 +313,15 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
 		state->magic_pending = false;
 	}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	/* Write VAD flag as first value after magic (as two int16_t = one int32_t) */
+	if (state->vad_pending && sink_samples >= 2) {
+		w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, 2, (int16_t *)&state->vad_flag);
+		sink_samples -= 2;
+		state->vad_pending = false;
+	}
+#endif
+
 	/* Write cepstral/mel data from scratch buffer */
 	to_copy = MIN(state->out_remain, sink_samples);
 	if (to_copy > 0) {
@@ -386,6 +407,9 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
 				state->mel_log_32[k] >>= 8;
 
 			state->out_data_ptr_32 = state->mel_log_32;
+#ifdef CONFIG_COMP_MFCC_VAD
+			state->vad_pending = true;
+#endif
 		} else {
 			state->out_data_ptr = state->cepstral_coef->data;
 		}
@@ -404,6 +428,15 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
 		state->magic_pending = false;
 	}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	/* Write VAD flag as first value after magic */
+	if (state->vad_pending && sink_samples >= 1) {
+		w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, 1, &state->vad_flag);
+		sink_samples -= 1;
+		state->vad_pending = false;
+	}
+#endif
+
 	if (state->mel_only) {
 		/* Write 32-bit mel data Q9.15, one value per int32_t */
 		to_copy = MIN(state->out_remain, sink_samples);
@@ -461,6 +494,9 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
 	if (num_ceps > 0) {
 		if (state->mel_only) {
 			state->out_data_ptr_32 = state->mel_log_32;
+#ifdef CONFIG_COMP_MFCC_VAD
+			state->vad_pending = true;
+#endif
 		} else {
 			state->out_data_ptr = state->cepstral_coef->data;
 		}
@@ -479,6 +515,15 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
 		state->magic_pending = false;
 	}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	/* Write VAD flag as first value after magic */
+	if (state->vad_pending && sink_samples >= 1) {
+		w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, 1, &state->vad_flag);
+		sink_samples -= 1;
+		state->vad_pending = false;
+	}
+#endif
+
 	if (state->mel_only) {
 		/* Write 32-bit mel data Q9.23, one value per int32_t */
 		to_copy = MIN(state->out_remain, sink_samples);

src/audio/mfcc/mfcc_setup.c

@@ -18,6 +18,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef CONFIG_COMP_MFCC_VAD
+#include <sof/audio/mfcc/mfcc_vad.h>
+#endif
+
 /* Definitions for cepstral lifter */
 #define PI_Q23 Q_CONVERT_FLOAT(3.1415926536, 23)
 #define TWO_PI_Q23 Q_CONVERT_FLOAT(6.2831853072, 23)
@@ -346,10 +350,22 @@ int mfcc_setup(struct processing_module *mod, int max_frames, int sample_rate, i
 	state->waiting_fill = true;
 	state->prev_samples_valid = false;
 	state->magic_pending = false;
+#ifdef CONFIG_COMP_MFCC_VAD
+	state->vad_pending = false;
+	state->vad_flag = 0;
+#endif
 	state->out_data_ptr = NULL;
 	state->out_data_ptr_32 = NULL;
 	state->out_remain = 0;
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	ret = mfcc_vad_init(&cd->vad, config->num_mel_bins, sample_rate, mod);
+	if (ret < 0) {
+		comp_err(dev, "Failed VAD init");
+		goto free_lifter;
+	}
+#endif
+
 	comp_dbg(dev, "done");
 	return 0;
 
@@ -389,4 +405,8 @@ void mfcc_free_buffers(struct processing_module *mod)
 	mod_free(mod, cd->state.melfb.data);
 	mod_free(mod, cd->state.dct.matrix);
 	mod_free(mod, cd->state.lifter.matrix);
+#ifdef CONFIG_COMP_MFCC_VAD
+	mod_free(mod, cd->vad.noise_floor);
+	mod_free(mod, cd->vad.weights);
+#endif
 }