Audio: MFCC: Add Voice Activity Detection based on Mel spectrum

This patch adds a new mfcc_vad module that implements VAD operating
on the Mel log spectrum values produced by the MFCC component. The
VAD is very simple and is not very selective for voice vs. other
signals. But the continuously updated background noise estimate
prevents stationary noises to trigger the VAD.

The algorithm tracks a per-bin noise floor (instant-down, slow-rise)
and computes a A-weighted energy delta. The used weight emphasizes
speech frequencies. Speech is declared when the delta exceeds a
threshold (0.30 in Q9.23) with a 20-frame hangover to prevent rapid
toggling.

The VAD flag is inserted into the output stream as the first value
after the magic header word in all format paths (S16, S24, S32).

A new Kconfig option CONFIG_COMP_MFCC_VAD (depends on COMP_MFCC,
default n) gates compilation of the VAD code and the stream format
change.

Signed-off-by: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>

Author: singalsu

src/arch/host/configs/library_defconfig

@@ -11,6 +11,7 @@ CONFIG_COMP_IIR=y
 CONFIG_COMP_IGO_NR=y
 CONFIG_COMP_LEVEL_MULTIPLIER=y
 CONFIG_COMP_MFCC=y
+CONFIG_COMP_MFCC_VAD=y
 CONFIG_COMP_MODULE_ADAPTER=y
 CONFIG_COMP_MULTIBAND_DRC=y
 CONFIG_COMP_MUX=y

src/audio/mfcc/CMakeLists.txt

@@ -5,4 +5,7 @@ if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
   add_dependencies(app mfcc)
 else()
   add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c)
+  if(CONFIG_COMP_MFCC_VAD)
+    add_local_sources(sof mfcc_vad.c)
+  endif()
 endif()

src/audio/mfcc/Kconfig

@@ -24,3 +24,14 @@ config COMP_MFCC
 	  The characteristic of the audio features are defined in the binary
 	  control blob. Directory tools/tune/mfcc contains a tool to create
 	  the configurations.
+
+config COMP_MFCC_VAD
+	bool "MFCC Voice Activity Detection"
+	depends on COMP_MFCC
+	default n
+	help
+	  This option enables a Voice Activity Detector (VAD) that operates
+	  on the Mel spectrum values produced by the MFCC component. The VAD
+	  flag is inserted into the output stream as the first int32_t value
+	  after the magic header word. The VAD tracks a per-bin noise floor
+	  and detects speech using a weighted energy delta with hangover.

src/audio/mfcc/mfcc_common.c

@@ -21,6 +21,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef CONFIG_COMP_MFCC_VAD
+#include <sof/audio/mfcc/mfcc_vad.h>
+#endif
+
 LOG_MODULE_REGISTER(mfcc_common, CONFIG_SOF_LOG_LEVEL);
 
 /*
@@ -144,6 +148,10 @@ static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *
 					sat_int32(Q_MULTSR_32X32(s, config->mel_scale, 23, 12, 23));
 			}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+			/* Run VAD on the mel log spectrum before further processing */
+			state->vad_flag = mfcc_vad_update(&cd->vad, state->mel_log_32);
+#endif
 			/* Store Q9.7 version in mel_spectra for s16 output mode */
 			for (j = 0; j < state->dct.num_in; j++)
 				state->mel_spectra->data[j] =
@@ -289,6 +297,9 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
 
 		state->out_remain = num_ceps;
 		state->magic_pending = true;
+#ifdef CONFIG_COMP_MFCC_VAD
+		state->vad_pending = true;
+#endif
 	}
 
 	/* Write to sink, limited by period size */
@@ -301,6 +312,15 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
 		state->magic_pending = false;
 	}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	/* Write VAD flag as first value after magic (as two int16_t = one int32_t) */
+	if (state->vad_pending && sink_samples >= 2) {
+		w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, 2, (int16_t *)&state->vad_flag);
+		sink_samples -= 2;
+		state->vad_pending = false;
+	}
+#endif
+
 	/* Write cepstral/mel data from scratch buffer */
 	to_copy = MIN(state->out_remain, sink_samples);
 	if (to_copy > 0) {
@@ -392,6 +412,9 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
 
 		state->out_remain = num_ceps;
 		state->magic_pending = true;
+#ifdef CONFIG_COMP_MFCC_VAD
+		state->vad_pending = true;
+#endif
 	}
 
 	/* Write to sink, limited by period size */
@@ -404,6 +427,15 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
 		state->magic_pending = false;
 	}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	/* Write VAD flag as first value after magic */
+	if (state->vad_pending && sink_samples >= 1) {
+		w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, 1, &state->vad_flag);
+		sink_samples -= 1;
+		state->vad_pending = false;
+	}
+#endif
+
 	if (state->mel_only) {
 		/* Write 32-bit mel data Q9.15, one value per int32_t */
 		to_copy = MIN(state->out_remain, sink_samples);
@@ -467,6 +499,9 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
 
 		state->out_remain = num_ceps;
 		state->magic_pending = true;
+#ifdef CONFIG_COMP_MFCC_VAD
+		state->vad_pending = true;
+#endif
 	}
 
 	/* Write to sink, limited by period size */
@@ -479,6 +514,15 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
 		state->magic_pending = false;
 	}
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	/* Write VAD flag as first value after magic */
+	if (state->vad_pending && sink_samples >= 1) {
+		w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, 1, &state->vad_flag);
+		sink_samples -= 1;
+		state->vad_pending = false;
+	}
+#endif
+
 	if (state->mel_only) {
 		/* Write 32-bit mel data Q9.23, one value per int32_t */
 		to_copy = MIN(state->out_remain, sink_samples);

src/audio/mfcc/mfcc_setup.c

@@ -18,6 +18,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef CONFIG_COMP_MFCC_VAD
+#include <sof/audio/mfcc/mfcc_vad.h>
+#endif
+
 /* Definitions for cepstral lifter */
 #define PI_Q23 Q_CONVERT_FLOAT(3.1415926536, 23)
 #define TWO_PI_Q23 Q_CONVERT_FLOAT(6.2831853072, 23)
@@ -346,10 +350,24 @@ int mfcc_setup(struct processing_module *mod, int max_frames, int sample_rate, i
 	state->waiting_fill = true;
 	state->prev_samples_valid = false;
 	state->magic_pending = false;
+#ifdef CONFIG_COMP_MFCC_VAD
+	state->vad_pending = false;
+	state->vad_flag = 0;
+#endif
 	state->out_data_ptr = NULL;
 	state->out_data_ptr_32 = NULL;
 	state->out_remain = 0;
 
+#ifdef CONFIG_COMP_MFCC_VAD
+	ret = mfcc_vad_init(&cd->vad, config->num_mel_bins, sample_rate);
+	if (ret < 0) {
+		comp_err(dev, "Failed VAD init");
+		goto free_lifter;
+	}
+
+	comp_info(dev, "VAD enabled, num_mel_bins = %d", config->num_mel_bins);
+#endif
+
 	comp_dbg(dev, "done");
 	return 0;