Skip to content

Commit 6583589

Browse files
lrgirdwolrgirdwo
committed
audio: tflm: add keyword detection logging, host notification, and HDA capture topology
1 parent 8c0c11b commit 6583589

5 files changed

Lines changed: 498 additions & 0 deletions

File tree

‎src/audio/tensorflow/tflm-classify.c‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@
3232
#include <stddef.h>
3333
#include <stdint.h>
3434

35+
#include <ipc4/base-config.h>
36+
#include <ipc4/header.h>
37+
#include <ipc4/module.h>
38+
#include <ipc4/notification.h>
39+
3540
#include "speech.h"
3641

3742
SOF_DEFINE_REG_UUID(tflmcly);
@@ -44,8 +49,67 @@ static const char * const prediction[] = TFLM_CATEGORY_DATA;
4449
struct tflm_comp_data {
4550
struct comp_data_blob_handler *model_handler;
4651
struct tf_classify tfc;
52+
struct ipc_msg *msg;
4753
};
4854

55+
static int tflm_ipc_notification_init(struct processing_module *mod)
56+
{
57+
struct tflm_comp_data *cd = module_get_private_data(mod);
58+
struct ipc_msg msg_proto;
59+
struct comp_dev *dev = mod->dev;
60+
struct comp_ipc_config *ipc_config = &dev->ipc_config;
61+
union ipc4_notification_header *primary =
62+
(union ipc4_notification_header *)&msg_proto.header;
63+
struct sof_ipc4_notify_module_data *msg_module_data;
64+
struct sof_ipc4_control_msg_payload *msg_payload;
65+
66+
memset_s(&msg_proto, sizeof(msg_proto), 0, sizeof(msg_proto));
67+
primary->r.notif_type = SOF_IPC4_MODULE_NOTIFICATION;
68+
primary->r.type = SOF_IPC4_GLB_NOTIFICATION;
69+
primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST;
70+
primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG;
71+
cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension,
72+
sizeof(struct sof_ipc4_notify_module_data) +
73+
sizeof(struct sof_ipc4_control_msg_payload) +
74+
sizeof(struct sof_ipc4_ctrl_value_chan));
75+
if (!cd->msg) {
76+
comp_err(dev, "Failed to initialize TFLM notification");
77+
return -ENOMEM;
78+
}
79+
80+
msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data;
81+
msg_module_data->instance_id = IPC4_INST_ID(ipc_config->id);
82+
msg_module_data->module_id = IPC4_MOD_ID(ipc_config->id);
83+
msg_module_data->event_id = SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL |
84+
SOF_IPC4_SWITCH_CONTROL_PARAM_ID;
85+
msg_module_data->event_data_size = sizeof(struct sof_ipc4_control_msg_payload) +
86+
sizeof(struct sof_ipc4_ctrl_value_chan);
87+
88+
msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data;
89+
msg_payload->id = 0;
90+
msg_payload->num_elems = 1;
91+
msg_payload->chanv[0].channel = 0;
92+
93+
comp_dbg(dev, "TFLM notification init: instance_id = 0x%08x, module_id = 0x%08x",
94+
msg_module_data->instance_id, msg_module_data->module_id);
95+
return 0;
96+
}
97+
98+
static void tflm_send_keyword_notification(struct processing_module *mod, uint32_t category_idx)
99+
{
100+
struct tflm_comp_data *cd = module_get_private_data(mod);
101+
struct sof_ipc4_notify_module_data *msg_module_data;
102+
struct sof_ipc4_control_msg_payload *msg_payload;
103+
104+
if (!cd->msg)
105+
return;
106+
107+
msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data;
108+
msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data;
109+
msg_payload->chanv[0].value = category_idx;
110+
ipc_msg_send(cd->msg, NULL, false);
111+
}
112+
49113
__cold static int tflm_init(struct processing_module *mod)
50114
{
51115
struct module_data *md = &mod->priv;
@@ -97,11 +161,19 @@ __cold static int tflm_init(struct processing_module *mod)
97161
goto fail;
98162
}
99163

164+
ret = tflm_ipc_notification_init(mod);
165+
if (ret < 0) {
166+
comp_err(dev, "failed to init notification");
167+
goto fail;
168+
}
169+
100170
return ret;
101171

102172
fail:
103173
/* Passing NULL pointer to free functions is Ok */
104174
mod_data_blob_handler_free(mod, cd->model_handler);
175+
if (cd->msg)
176+
ipc_msg_free(cd->msg);
105177
mod_free(mod, cd);
106178
return ret;
107179
}
@@ -112,6 +184,8 @@ __cold static int tflm_free(struct processing_module *mod)
112184

113185
assert_can_be_cold();
114186

187+
if (cd->msg)
188+
ipc_msg_free(cd->msg);
115189
mod_data_blob_handler_free(mod, cd->model_handler);
116190
mod_free(mod, cd);
117191
return 0;
@@ -208,9 +282,23 @@ static int tflm_process(struct processing_module *mod,
208282
}
209283

210284
/* debug - dump the output */
285+
int max_idx = 0;
286+
float max_score = cd->tfc.predictions[0];
287+
211288
for (int i = 0; i < cd->tfc.categories; i++) {
212289
comp_dbg(dev, "tf: predictions %1.3f %s",
213290
cd->tfc.predictions[i], prediction[i]);
291+
if (cd->tfc.predictions[i] > max_score) {
292+
max_score = cd->tfc.predictions[i];
293+
max_idx = i;
294+
}
295+
}
296+
297+
/* Check if a keyword ("yes" or "no", category indices 2 and 3) is detected */
298+
if (max_idx >= 2 && max_score >= 0.70f) {
299+
comp_info(dev, "TFLM keyword detected: %s (confidence %1.3f)",
300+
prediction[max_idx], (double)max_score);
301+
tflm_send_keyword_notification(mod, max_idx);
214302
}
215303

216304
/* advance by one stride */

‎tools/topology/topology2/include/components/tflm.conf‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#
2+
# A TFLM Classification component for SOF. All attributes defined herein are namespaced
3+
# by alsatplg to "Object.Widget.tflmcly.attribute_name"
4+
#
5+
6+
Class.Widget."tflmcly" {
7+
#
8+
# Pipeline ID
9+
#
10+
DefineAttribute."index" {
11+
type "integer"
12+
}
13+
14+
#
15+
# Unique instance for TFLM widget
16+
#
17+
DefineAttribute."instance" {
18+
type "integer"
19+
}
20+
21+
# Include common widget attributes definition
22+
<include/components/widget-common.conf>
23+
24+
attributes {
25+
!constructor [
26+
"index"
27+
"instance"
28+
]
29+
!mandatory [
30+
"num_input_pins"
31+
"num_output_pins"
32+
"num_input_audio_formats"
33+
"num_output_audio_formats"
34+
]
35+
36+
!immutable [
37+
"uuid"
38+
]
39+
!deprecated [
40+
"preload_count"
41+
]
42+
unique "instance"
43+
}
44+
45+
#
46+
# Default attributes for tflmcly
47+
# UUID: c51dc642-a2e1-48df-a490e2748cb6363e
48+
#
49+
uuid "42:c6:1d:c5:e1:a2:df:48:a4:90:e2:74:8c:b6:36:3e"
50+
type "effect"
51+
no_pm "true"
52+
num_input_pins 1
53+
num_output_pins 1
54+
}

‎tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
#
2+
# SRC-MFCC-TFLM capture pipeline
3+
#
4+
# This class provides host pipeline for capture with MFCC feature extraction
5+
# fed to TFLM classification micro speech module.
6+
#
7+
8+
<include/common/input_audio_format.conf>
9+
<include/common/output_audio_format.conf>
10+
<include/components/pipeline.conf>
11+
<include/components/host-copier.conf>
12+
<include/components/src.conf>
13+
<include/components/mfcc.conf>
14+
<include/components/tflm.conf>
15+
16+
Define {
17+
MFCC_FRAME_BYTES 344
18+
}
19+
20+
Class.Pipeline."host-gateway-src-mfcc-tflm-capture" {
21+
22+
<include/pipelines/pipeline-common.conf>
23+
24+
attributes {
25+
!constructor [
26+
"index"
27+
]
28+
29+
unique "instance"
30+
}
31+
32+
Object.Widget {
33+
src."1" {
34+
num_input_pins 1
35+
num_output_pins 1
36+
num_input_audio_formats 3
37+
num_output_audio_formats 1
38+
Object.Base.input_audio_format [
39+
{
40+
in_bit_depth 32
41+
in_valid_bit_depth 32
42+
in_rate 48000
43+
}
44+
{
45+
in_bit_depth 32
46+
in_valid_bit_depth 32
47+
in_rate 96000
48+
}
49+
{
50+
in_bit_depth 32
51+
in_valid_bit_depth 32
52+
in_rate 192000
53+
}
54+
]
55+
Object.Base.output_audio_format [
56+
{
57+
out_bit_depth 32
58+
out_valid_bit_depth 32
59+
out_rate 16000
60+
}
61+
]
62+
}
63+
64+
mfcc."1" {
65+
Object.Control {
66+
mixer."1" {
67+
name "HDA Mic MFCC VAD"
68+
}
69+
}
70+
num_input_audio_formats 1
71+
num_output_audio_formats 1
72+
Object.Base.input_audio_format [
73+
{
74+
in_bit_depth 32
75+
in_valid_bit_depth 32
76+
in_rate 16000
77+
}
78+
]
79+
Object.Base.output_audio_format [
80+
{
81+
out_bit_depth 32
82+
out_valid_bit_depth 32
83+
out_rate 16000
84+
obs $MFCC_FRAME_BYTES
85+
}
86+
]
87+
}
88+
89+
tflmcly."1" {
90+
num_input_audio_formats 1
91+
num_output_audio_formats 1
92+
Object.Base.input_audio_format [
93+
{
94+
in_bit_depth 32
95+
in_valid_bit_depth 32
96+
in_rate 16000
97+
ibs $MFCC_FRAME_BYTES
98+
}
99+
]
100+
Object.Base.output_audio_format [
101+
{
102+
out_bit_depth 32
103+
out_valid_bit_depth 32
104+
out_rate 16000
105+
obs $MFCC_FRAME_BYTES
106+
}
107+
]
108+
}
109+
110+
host-copier."1" {
111+
type "aif_out"
112+
node_type $HDA_HOST_INPUT_CLASS
113+
num_input_pins 1
114+
num_output_pins 1
115+
num_input_audio_formats 1
116+
num_output_audio_formats 1
117+
Object.Base.input_audio_format [
118+
{
119+
in_bit_depth 32
120+
in_valid_bit_depth 32
121+
in_rate 16000
122+
ibs $MFCC_FRAME_BYTES
123+
}
124+
]
125+
Object.Base.output_audio_format [
126+
{
127+
out_bit_depth 32
128+
out_valid_bit_depth 32
129+
out_rate 16000
130+
obs $MFCC_FRAME_BYTES
131+
}
132+
]
133+
}
134+
135+
pipeline."1" {
136+
priority 0
137+
lp_mode 0
138+
}
139+
}
140+
141+
Object.Base {
142+
!route [
143+
{
144+
source src.$index.1
145+
sink mfcc.$index.1
146+
}
147+
{
148+
source mfcc.$index.1
149+
sink tflmcly.$index.1
150+
}
151+
]
152+
}
153+
154+
direction "capture"
155+
dynamic_pipeline 1
156+
time_domain "timer"
157+
}

‎tools/topology/topology2/production/tplg-targets-hda-generic.cmake‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,4 +123,7 @@ EFX_DMIC0_TDFB_PARAMS=line4_pass,EFX_DMIC0_DRC_PARAMS=dmic_default"
123123
PREPROCESS_PLUGINS=nhlt,NHLT_BIN=nhlt-sof-hda-generic-ace3-2ch-dax.bin,\
124124
DMIC0_ENHANCED_CAPTURE=true,EFX_DMIC0_TDFB_PARAMS=line2_generic_pm10deg,\
125125
EFX_DMIC0_DRC_PARAMS=dmic_default,DOLBY_DAX_CORE_ID=1"
126+
127+
# HDA Mic TFLM Topology Target
128+
"sof-hda-tflm\;sof-hda-tflm\;"
126129
)

0 commit comments

Comments
 (0)