From 65835891cad0da459397207f7ea93c3f437f5165 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Wed, 22 Jul 2026 21:07:06 +0100 Subject: [PATCH 01/14] audio: tflm: add keyword detection logging, host notification, and HDA capture topology --- src/audio/tensorflow/tflm-classify.c | 88 ++++++++ .../topology2/include/components/tflm.conf | 54 +++++ .../host-gateway-src-mfcc-tflm-capture.conf | 157 ++++++++++++++ .../production/tplg-targets-hda-generic.cmake | 3 + tools/topology/topology2/sof-hda-tflm.conf | 196 ++++++++++++++++++ 5 files changed, 498 insertions(+) create mode 100644 tools/topology/topology2/include/components/tflm.conf create mode 100644 tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf create mode 100644 tools/topology/topology2/sof-hda-tflm.conf diff --git a/src/audio/tensorflow/tflm-classify.c b/src/audio/tensorflow/tflm-classify.c index 9c93329bce1c..b12017a76dc0 100644 --- a/src/audio/tensorflow/tflm-classify.c +++ b/src/audio/tensorflow/tflm-classify.c @@ -32,6 +32,11 @@ #include #include +#include +#include +#include +#include + #include "speech.h" SOF_DEFINE_REG_UUID(tflmcly); @@ -44,8 +49,67 @@ static const char * const prediction[] = TFLM_CATEGORY_DATA; struct tflm_comp_data { struct comp_data_blob_handler *model_handler; struct tf_classify tfc; + struct ipc_msg *msg; }; +static int tflm_ipc_notification_init(struct processing_module *mod) +{ + struct tflm_comp_data *cd = module_get_private_data(mod); + struct ipc_msg msg_proto; + struct comp_dev *dev = mod->dev; + struct comp_ipc_config *ipc_config = &dev->ipc_config; + union ipc4_notification_header *primary = + (union ipc4_notification_header *)&msg_proto.header; + struct sof_ipc4_notify_module_data *msg_module_data; + struct sof_ipc4_control_msg_payload *msg_payload; + + memset_s(&msg_proto, sizeof(msg_proto), 0, sizeof(msg_proto)); + primary->r.notif_type = SOF_IPC4_MODULE_NOTIFICATION; + primary->r.type = SOF_IPC4_GLB_NOTIFICATION; + primary->r.rsp = SOF_IPC4_MESSAGE_DIR_MSG_REQUEST; + primary->r.msg_tgt = SOF_IPC4_MESSAGE_TARGET_FW_GEN_MSG; + cd->msg = ipc_msg_w_ext_init(msg_proto.header, msg_proto.extension, + sizeof(struct sof_ipc4_notify_module_data) + + sizeof(struct sof_ipc4_control_msg_payload) + + sizeof(struct sof_ipc4_ctrl_value_chan)); + if (!cd->msg) { + comp_err(dev, "Failed to initialize TFLM notification"); + return -ENOMEM; + } + + msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data; + msg_module_data->instance_id = IPC4_INST_ID(ipc_config->id); + msg_module_data->module_id = IPC4_MOD_ID(ipc_config->id); + msg_module_data->event_id = SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL | + SOF_IPC4_SWITCH_CONTROL_PARAM_ID; + msg_module_data->event_data_size = sizeof(struct sof_ipc4_control_msg_payload) + + sizeof(struct sof_ipc4_ctrl_value_chan); + + msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data; + msg_payload->id = 0; + msg_payload->num_elems = 1; + msg_payload->chanv[0].channel = 0; + + comp_dbg(dev, "TFLM notification init: instance_id = 0x%08x, module_id = 0x%08x", + msg_module_data->instance_id, msg_module_data->module_id); + return 0; +} + +static void tflm_send_keyword_notification(struct processing_module *mod, uint32_t category_idx) +{ + struct tflm_comp_data *cd = module_get_private_data(mod); + struct sof_ipc4_notify_module_data *msg_module_data; + struct sof_ipc4_control_msg_payload *msg_payload; + + if (!cd->msg) + return; + + msg_module_data = (struct sof_ipc4_notify_module_data *)cd->msg->tx_data; + msg_payload = (struct sof_ipc4_control_msg_payload *)msg_module_data->event_data; + msg_payload->chanv[0].value = category_idx; + ipc_msg_send(cd->msg, NULL, false); +} + __cold static int tflm_init(struct processing_module *mod) { struct module_data *md = &mod->priv; @@ -97,11 +161,19 @@ __cold static int tflm_init(struct processing_module *mod) goto fail; } + ret = tflm_ipc_notification_init(mod); + if (ret < 0) { + comp_err(dev, "failed to init notification"); + goto fail; + } + return ret; fail: /* Passing NULL pointer to free functions is Ok */ mod_data_blob_handler_free(mod, cd->model_handler); + if (cd->msg) + ipc_msg_free(cd->msg); mod_free(mod, cd); return ret; } @@ -112,6 +184,8 @@ __cold static int tflm_free(struct processing_module *mod) assert_can_be_cold(); + if (cd->msg) + ipc_msg_free(cd->msg); mod_data_blob_handler_free(mod, cd->model_handler); mod_free(mod, cd); return 0; @@ -208,9 +282,23 @@ static int tflm_process(struct processing_module *mod, } /* debug - dump the output */ + int max_idx = 0; + float max_score = cd->tfc.predictions[0]; + for (int i = 0; i < cd->tfc.categories; i++) { comp_dbg(dev, "tf: predictions %1.3f %s", cd->tfc.predictions[i], prediction[i]); + if (cd->tfc.predictions[i] > max_score) { + max_score = cd->tfc.predictions[i]; + max_idx = i; + } + } + + /* Check if a keyword ("yes" or "no", category indices 2 and 3) is detected */ + if (max_idx >= 2 && max_score >= 0.70f) { + comp_info(dev, "TFLM keyword detected: %s (confidence %1.3f)", + prediction[max_idx], (double)max_score); + tflm_send_keyword_notification(mod, max_idx); } /* advance by one stride */ diff --git a/tools/topology/topology2/include/components/tflm.conf b/tools/topology/topology2/include/components/tflm.conf new file mode 100644 index 000000000000..6144b2ce91de --- /dev/null +++ b/tools/topology/topology2/include/components/tflm.conf @@ -0,0 +1,54 @@ +# +# A TFLM Classification component for SOF. All attributes defined herein are namespaced +# by alsatplg to "Object.Widget.tflmcly.attribute_name" +# + +Class.Widget."tflmcly" { + # + # Pipeline ID + # + DefineAttribute."index" { + type "integer" + } + + # + # Unique instance for TFLM widget + # + DefineAttribute."instance" { + type "integer" + } + + # Include common widget attributes definition + + + attributes { + !constructor [ + "index" + "instance" + ] + !mandatory [ + "num_input_pins" + "num_output_pins" + "num_input_audio_formats" + "num_output_audio_formats" + ] + + !immutable [ + "uuid" + ] + !deprecated [ + "preload_count" + ] + unique "instance" + } + + # + # Default attributes for tflmcly + # UUID: c51dc642-a2e1-48df-a490e2748cb6363e + # + uuid "42:c6:1d:c5:e1:a2:df:48:a4:90:e2:74:8c:b6:36:3e" + type "effect" + no_pm "true" + num_input_pins 1 + num_output_pins 1 +} diff --git a/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf b/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf new file mode 100644 index 000000000000..47db5e1c80ba --- /dev/null +++ b/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf @@ -0,0 +1,157 @@ +# +# SRC-MFCC-TFLM capture pipeline +# +# This class provides host pipeline for capture with MFCC feature extraction +# fed to TFLM classification micro speech module. +# + + + + + + + + + +Define { + MFCC_FRAME_BYTES 344 +} + +Class.Pipeline."host-gateway-src-mfcc-tflm-capture" { + + + + attributes { + !constructor [ + "index" + ] + + unique "instance" + } + + Object.Widget { + src."1" { + num_input_pins 1 + num_output_pins 1 + num_input_audio_formats 3 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_bit_depth 32 + in_valid_bit_depth 32 + in_rate 48000 + } + { + in_bit_depth 32 + in_valid_bit_depth 32 + in_rate 96000 + } + { + in_bit_depth 32 + in_valid_bit_depth 32 + in_rate 192000 + } + ] + Object.Base.output_audio_format [ + { + out_bit_depth 32 + out_valid_bit_depth 32 + out_rate 16000 + } + ] + } + + mfcc."1" { + Object.Control { + mixer."1" { + name "HDA Mic MFCC VAD" + } + } + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_bit_depth 32 + in_valid_bit_depth 32 + in_rate 16000 + } + ] + Object.Base.output_audio_format [ + { + out_bit_depth 32 + out_valid_bit_depth 32 + out_rate 16000 + obs $MFCC_FRAME_BYTES + } + ] + } + + tflmcly."1" { + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_bit_depth 32 + in_valid_bit_depth 32 + in_rate 16000 + ibs $MFCC_FRAME_BYTES + } + ] + Object.Base.output_audio_format [ + { + out_bit_depth 32 + out_valid_bit_depth 32 + out_rate 16000 + obs $MFCC_FRAME_BYTES + } + ] + } + + host-copier."1" { + type "aif_out" + node_type $HDA_HOST_INPUT_CLASS + num_input_pins 1 + num_output_pins 1 + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_bit_depth 32 + in_valid_bit_depth 32 + in_rate 16000 + ibs $MFCC_FRAME_BYTES + } + ] + Object.Base.output_audio_format [ + { + out_bit_depth 32 + out_valid_bit_depth 32 + out_rate 16000 + obs $MFCC_FRAME_BYTES + } + ] + } + + pipeline."1" { + priority 0 + lp_mode 0 + } + } + + Object.Base { + !route [ + { + source src.$index.1 + sink mfcc.$index.1 + } + { + source mfcc.$index.1 + sink tflmcly.$index.1 + } + ] + } + + direction "capture" + dynamic_pipeline 1 + time_domain "timer" +} diff --git a/tools/topology/topology2/production/tplg-targets-hda-generic.cmake b/tools/topology/topology2/production/tplg-targets-hda-generic.cmake index 72872d110a4d..3e9b360b2ee9 100644 --- a/tools/topology/topology2/production/tplg-targets-hda-generic.cmake +++ b/tools/topology/topology2/production/tplg-targets-hda-generic.cmake @@ -123,4 +123,7 @@ EFX_DMIC0_TDFB_PARAMS=line4_pass,EFX_DMIC0_DRC_PARAMS=dmic_default" PREPROCESS_PLUGINS=nhlt,NHLT_BIN=nhlt-sof-hda-generic-ace3-2ch-dax.bin,\ DMIC0_ENHANCED_CAPTURE=true,EFX_DMIC0_TDFB_PARAMS=line2_generic_pm10deg,\ EFX_DMIC0_DRC_PARAMS=dmic_default,DOLBY_DAX_CORE_ID=1" + +# HDA Mic TFLM Topology Target +"sof-hda-tflm\;sof-hda-tflm\;" ) diff --git a/tools/topology/topology2/sof-hda-tflm.conf b/tools/topology/topology2/sof-hda-tflm.conf new file mode 100644 index 000000000000..e1160a998732 --- /dev/null +++ b/tools/topology/topology2/sof-hda-tflm.conf @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Define { + HDA_CONFIG "none" + PLATFORM "none" + ANALOG_PLAYBACK_PCM 'Analog Playback' + ANALOG_CAPTURE_PCM 'HDA Mic TFLM Capture' + HDA_ANALOG_DAI_NAME 'Analog' + HDA_ANALOG_CAPTURE_RATE 48000 + HDA_ANALOG_PLAYBACK_RATE 48000 + HDA_TFLM_CAPTURE_PCM_ID 0 + HDA_TFLM_HOST_PIPELINE_ID 1 + HDA_TFLM_DAI_PIPELINE_ID 2 +} + +# Override defaults with platform-specific config if set +IncludeByKey.PLATFORM { + "mtl" "platform/intel/mtl.conf" + "lnl" "platform/intel/lnl.conf" + "ptl" "platform/intel/ptl.conf" +} + +Object.Dai.HDA [ + { + name $HDA_ANALOG_DAI_NAME + dai_index 0 + id 4 + default_hw_conf_id 4 + Object.Base.hw_config.1 { + name "HDA0" + } + direction duplex + } +] + +Object.Pipeline { + host-gateway-src-mfcc-tflm-capture [ + { + index $HDA_TFLM_HOST_PIPELINE_ID + + Object.Widget.host-copier.1 { + stream_name $ANALOG_CAPTURE_PCM + pcm_id $HDA_TFLM_CAPTURE_PCM_ID + } + } + ] + + dai-copier-gain-module-copier-capture [ + { + index $HDA_TFLM_DAI_PIPELINE_ID + + Object.Widget.dai-copier.1 { + dai_type "HDA" + type "dai_out" + copier_type "HDA" + stream_name $HDA_ANALOG_DAI_NAME + node_type $HDA_LINK_INPUT_CLASS + num_output_pins 1 + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_rate $HDA_ANALOG_CAPTURE_RATE + in_bit_depth 32 + in_valid_bit_depth 32 + } + ] + Object.Base.output_audio_format [ + { + out_rate $HDA_ANALOG_CAPTURE_RATE + out_bit_depth 32 + out_valid_bit_depth 32 + } + ] + } + + Object.Widget.gain.1 { + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_rate $HDA_ANALOG_CAPTURE_RATE + in_bit_depth 32 + in_valid_bit_depth 32 + } + ] + Object.Base.output_audio_format [ + { + out_rate $HDA_ANALOG_CAPTURE_RATE + out_bit_depth 32 + out_valid_bit_depth 32 + } + ] + Object.Control.mixer.1 { + name 'HDA Mic Capture Volume' + } + } + + Object.Widget.module-copier.2 { + stream_name $HDA_ANALOG_DAI_NAME + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_rate $HDA_ANALOG_CAPTURE_RATE + in_bit_depth 32 + in_valid_bit_depth 32 + } + ] + Object.Base.output_audio_format [ + { + out_rate $HDA_ANALOG_CAPTURE_RATE + out_bit_depth 32 + out_valid_bit_depth 32 + } + ] + } + } + ] +} + +Object.Base.route [ + { + source "dai-copier.HDA.$HDA_ANALOG_DAI_NAME.capture" + sink "gain.$HDA_TFLM_DAI_PIPELINE_ID.1" + } + { + source "gain.$HDA_TFLM_DAI_PIPELINE_ID.1" + sink "module-copier.$HDA_TFLM_DAI_PIPELINE_ID.2" + } + { + source "module-copier.$HDA_TFLM_DAI_PIPELINE_ID.2" + sink "src.$HDA_TFLM_HOST_PIPELINE_ID.1" + } + { + source "tflmcly.$HDA_TFLM_HOST_PIPELINE_ID.1" + sink "host-copier.$HDA_TFLM_CAPTURE_PCM_ID.capture" + } +] + +Object.PCM.pcm [ + { + name "$ANALOG_CAPTURE_PCM" + id $HDA_TFLM_CAPTURE_PCM_ID + direction "capture" + + Object.Base.fe_dai.1 { + name "$ANALOG_CAPTURE_PCM" + } + + Object.PCM.pcm_caps.1 { + name $ANALOG_CAPTURE_PCM + formats 'S16_LE' + rates '16000' + channels_min 1 + channels_max 1 + } + } +] From 411ed5ed0c20b7422423641c311c11108f814c7c Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Wed, 22 Jul 2026 21:09:26 +0100 Subject: [PATCH 02/14] audio: tflm: fix tensor byte sizing for micro speech int8 input --- src/audio/tensorflow/tflm-classify.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/audio/tensorflow/tflm-classify.c b/src/audio/tensorflow/tflm-classify.c index b12017a76dc0..66136fac8cae 100644 --- a/src/audio/tensorflow/tflm-classify.c +++ b/src/audio/tensorflow/tflm-classify.c @@ -254,24 +254,23 @@ static int tflm_process(struct processing_module *mod, { struct tflm_comp_data *cd = module_get_private_data(mod); struct comp_dev *dev = mod->dev; - size_t frame_bytes = source_get_frame_bytes(sources[0]); - int features = source_get_data_frames_available(sources[0]); + int features_available = source_get_data_frames_available(sources[0]); const void *data_ptr, *buf_start; size_t buf_size; - int ret; + int ret = 0; comp_dbg(dev, "entry"); - /* Window size is TFLM_FEATURE_ELEM_COUNT and we increment - * by TFLM_FEATURE_SIZE until buffer empty. + /* Window size is TFLM_FEATURE_ELEM_COUNT (1960 bytes int8_t) and we increment + * by TFLM_FEATURE_SIZE (40 bytes int8_t) until buffer empty. */ - while (features >= TFLM_FEATURE_ELEM_COUNT) { - ret = source_get_data(sources[0], TFLM_FEATURE_ELEM_COUNT * frame_bytes, + while (features_available >= TFLM_FEATURE_ELEM_COUNT) { + ret = source_get_data(sources[0], TFLM_FEATURE_ELEM_COUNT, &data_ptr, &buf_start, &buf_size); if (ret) return ret; - cd->tfc.audio_features = data_ptr; + cd->tfc.audio_features = (int8_t *)data_ptr; cd->tfc.audio_data_size = TFLM_FEATURE_ELEM_COUNT; ret = TF_ProcessClassify(&cd->tfc); if (!ret) { @@ -281,7 +280,7 @@ static int tflm_process(struct processing_module *mod, return ret; } - /* debug - dump the output */ + /* Evaluate predictions */ int max_idx = 0; float max_score = cd->tfc.predictions[0]; @@ -301,9 +300,9 @@ static int tflm_process(struct processing_module *mod, tflm_send_keyword_notification(mod, max_idx); } - /* advance by one stride */ - source_release_data(sources[0], TFLM_FEATURE_SIZE * frame_bytes); - features = source_get_data_frames_available(sources[0]); + /* advance by one 20ms stride (40 int8_t features) */ + source_release_data(sources[0], TFLM_FEATURE_SIZE); + features_available = source_get_data_frames_available(sources[0]); } return ret; From 3776df67a223d20b3e1818b8edf2acc486fc1e91 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Wed, 22 Jul 2026 21:15:44 +0100 Subject: [PATCH 03/14] audio: tflm: integrate keyword detection with KPB for WoV trigger --- src/audio/tensorflow/tflm-classify.c | 23 +++++++++++++++++++ tools/topology/topology2/sof-hda-tflm.conf | 26 +++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/audio/tensorflow/tflm-classify.c b/src/audio/tensorflow/tflm-classify.c index 66136fac8cae..15fbe6e34d2a 100644 --- a/src/audio/tensorflow/tflm-classify.c +++ b/src/audio/tensorflow/tflm-classify.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include "speech.h" @@ -50,8 +52,28 @@ struct tflm_comp_data { struct comp_data_blob_handler *model_handler; struct tf_classify tfc; struct ipc_msg *msg; + struct kpb_event_data event_data; + struct kpb_client client_data; }; +static void tflm_notify_kpb(struct processing_module *mod) +{ + struct tflm_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + + comp_info(dev, "TFLM keyword trigger -> notifying KPB to begin draining"); + + cd->client_data.r_ptr = NULL; + cd->client_data.sink = NULL; + cd->client_data.id = 0; + cd->event_data.event_id = KPB_EVENT_BEGIN_DRAINING; + cd->event_data.client_data = &cd->client_data; + + notifier_event(dev, NOTIFIER_ID_KPB_CLIENT_EVT, + NOTIFIER_TARGET_CORE_ALL_MASK, &cd->event_data, + sizeof(cd->event_data)); +} + static int tflm_ipc_notification_init(struct processing_module *mod) { struct tflm_comp_data *cd = module_get_private_data(mod); @@ -298,6 +320,7 @@ static int tflm_process(struct processing_module *mod, comp_info(dev, "TFLM keyword detected: %s (confidence %1.3f)", prediction[max_idx], (double)max_score); tflm_send_keyword_notification(mod, max_idx); + tflm_notify_kpb(mod); } /* advance by one 20ms stride (40 int8_t features) */ diff --git a/tools/topology/topology2/sof-hda-tflm.conf b/tools/topology/topology2/sof-hda-tflm.conf index e1160a998732..ac8e8512ffc7 100644 --- a/tools/topology/topology2/sof-hda-tflm.conf +++ b/tools/topology/topology2/sof-hda-tflm.conf @@ -32,6 +32,7 @@ + @@ -41,7 +42,7 @@ Define { HDA_CONFIG "none" PLATFORM "none" ANALOG_PLAYBACK_PCM 'Analog Playback' - ANALOG_CAPTURE_PCM 'HDA Mic TFLM Capture' + ANALOG_CAPTURE_PCM 'HDA Mic TFLM WoV Capture' HDA_ANALOG_DAI_NAME 'Analog' HDA_ANALOG_CAPTURE_RATE 48000 HDA_ANALOG_PLAYBACK_RATE 48000 @@ -133,6 +134,25 @@ Object.Pipeline { } } + Object.Widget.kpb.1 { + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_rate $HDA_ANALOG_CAPTURE_RATE + in_bit_depth 32 + in_valid_bit_depth 32 + } + ] + Object.Base.output_audio_format [ + { + out_rate $HDA_ANALOG_CAPTURE_RATE + out_bit_depth 32 + out_valid_bit_depth 32 + } + ] + } + Object.Widget.module-copier.2 { stream_name $HDA_ANALOG_DAI_NAME num_input_audio_formats 1 @@ -163,6 +183,10 @@ Object.Base.route [ } { source "gain.$HDA_TFLM_DAI_PIPELINE_ID.1" + sink "kpb.$HDA_TFLM_DAI_PIPELINE_ID.1" + } + { + source "kpb.$HDA_TFLM_DAI_PIPELINE_ID.1" sink "module-copier.$HDA_TFLM_DAI_PIPELINE_ID.2" } { From a6c639cff89213611d444c89e941aecfb7b40ca1 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Wed, 22 Jul 2026 21:24:05 +0100 Subject: [PATCH 04/14] tools: topology2: update WoV pipeline graph with dual-path KPB routing --- .../host-gateway-src-mfcc-tflm-capture.conf | 39 ++++-------- tools/topology/topology2/sof-hda-tflm.conf | 61 ++++++++++--------- 2 files changed, 42 insertions(+), 58 deletions(-) diff --git a/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf b/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf index 47db5e1c80ba..841dd1630054 100644 --- a/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf +++ b/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf @@ -1,14 +1,13 @@ # -# SRC-MFCC-TFLM capture pipeline +# SRC-MFCC-TFLM capture detection pipeline # -# This class provides host pipeline for capture with MFCC feature extraction +# This class provides detection pipeline for capture with MFCC feature extraction # fed to TFLM classification micro speech module. # - @@ -30,6 +29,11 @@ Class.Pipeline."host-gateway-src-mfcc-tflm-capture" { } Object.Widget { + virtual."1" { + name "virtual.tflm_sink" + type out_drv + } + src."1" { num_input_pins 1 num_output_pins 1 @@ -107,31 +111,6 @@ Class.Pipeline."host-gateway-src-mfcc-tflm-capture" { ] } - host-copier."1" { - type "aif_out" - node_type $HDA_HOST_INPUT_CLASS - num_input_pins 1 - num_output_pins 1 - num_input_audio_formats 1 - num_output_audio_formats 1 - Object.Base.input_audio_format [ - { - in_bit_depth 32 - in_valid_bit_depth 32 - in_rate 16000 - ibs $MFCC_FRAME_BYTES - } - ] - Object.Base.output_audio_format [ - { - out_bit_depth 32 - out_valid_bit_depth 32 - out_rate 16000 - obs $MFCC_FRAME_BYTES - } - ] - } - pipeline."1" { priority 0 lp_mode 0 @@ -148,6 +127,10 @@ Class.Pipeline."host-gateway-src-mfcc-tflm-capture" { source mfcc.$index.1 sink tflmcly.$index.1 } + { + source tflmcly.$index.1 + sink "virtual.tflm_sink" + } ] } diff --git a/tools/topology/topology2/sof-hda-tflm.conf b/tools/topology/topology2/sof-hda-tflm.conf index ac8e8512ffc7..1c36c4d247ed 100644 --- a/tools/topology/topology2/sof-hda-tflm.conf +++ b/tools/topology/topology2/sof-hda-tflm.conf @@ -47,7 +47,8 @@ Define { HDA_ANALOG_CAPTURE_RATE 48000 HDA_ANALOG_PLAYBACK_RATE 48000 HDA_TFLM_CAPTURE_PCM_ID 0 - HDA_TFLM_HOST_PIPELINE_ID 1 + HDA_TFLM_HOST_PIPELINE_ID 0 + HDA_TFLM_DETECT_PIPELINE_ID 1 HDA_TFLM_DAI_PIPELINE_ID 2 } @@ -72,17 +73,39 @@ Object.Dai.HDA [ ] Object.Pipeline { - host-gateway-src-mfcc-tflm-capture [ + host-gateway-capture [ { index $HDA_TFLM_HOST_PIPELINE_ID Object.Widget.host-copier.1 { stream_name $ANALOG_CAPTURE_PCM pcm_id $HDA_TFLM_CAPTURE_PCM_ID + num_input_audio_formats 1 + num_output_audio_formats 1 + Object.Base.input_audio_format [ + { + in_rate $HDA_ANALOG_CAPTURE_RATE + in_bit_depth 32 + in_valid_bit_depth 32 + } + ] + Object.Base.output_audio_format [ + { + out_rate $HDA_ANALOG_CAPTURE_RATE + out_bit_depth 32 + out_valid_bit_depth 32 + } + ] } } ] + host-gateway-src-mfcc-tflm-capture [ + { + index $HDA_TFLM_DETECT_PIPELINE_ID + } + ] + dai-copier-gain-module-copier-capture [ { index $HDA_TFLM_DAI_PIPELINE_ID @@ -152,26 +175,6 @@ Object.Pipeline { } ] } - - Object.Widget.module-copier.2 { - stream_name $HDA_ANALOG_DAI_NAME - num_input_audio_formats 1 - num_output_audio_formats 1 - Object.Base.input_audio_format [ - { - in_rate $HDA_ANALOG_CAPTURE_RATE - in_bit_depth 32 - in_valid_bit_depth 32 - } - ] - Object.Base.output_audio_format [ - { - out_rate $HDA_ANALOG_CAPTURE_RATE - out_bit_depth 32 - out_valid_bit_depth 32 - } - ] - } } ] } @@ -185,16 +188,14 @@ Object.Base.route [ source "gain.$HDA_TFLM_DAI_PIPELINE_ID.1" sink "kpb.$HDA_TFLM_DAI_PIPELINE_ID.1" } + # KPB output pin 1 -> Real-time detection path (SRC -> MFCC -> TFLM) { source "kpb.$HDA_TFLM_DAI_PIPELINE_ID.1" - sink "module-copier.$HDA_TFLM_DAI_PIPELINE_ID.2" + sink "src.$HDA_TFLM_DETECT_PIPELINE_ID.1" } + # KPB output pin 2 -> Host WoV Draining PCM capture stream { - source "module-copier.$HDA_TFLM_DAI_PIPELINE_ID.2" - sink "src.$HDA_TFLM_HOST_PIPELINE_ID.1" - } - { - source "tflmcly.$HDA_TFLM_HOST_PIPELINE_ID.1" + source "kpb.$HDA_TFLM_DAI_PIPELINE_ID.1" sink "host-copier.$HDA_TFLM_CAPTURE_PCM_ID.capture" } ] @@ -211,8 +212,8 @@ Object.PCM.pcm [ Object.PCM.pcm_caps.1 { name $ANALOG_CAPTURE_PCM - formats 'S16_LE' - rates '16000' + formats 'S32_LE' + rates '48000' channels_min 1 channels_max 1 } From df47967326216420a38893b24c3f33753c178a8e Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Wed, 22 Jul 2026 21:26:24 +0100 Subject: [PATCH 05/14] docs: tensorflow: add comprehensive README for TFLM WoV architecture and topology usage --- src/audio/tensorflow/README.md | 156 ++++++++++++++++++++++++++++++--- 1 file changed, 144 insertions(+), 12 deletions(-) diff --git a/src/audio/tensorflow/README.md b/src/audio/tensorflow/README.md index a8e3f59ba3ce..7246f5d63c77 100644 --- a/src/audio/tensorflow/README.md +++ b/src/audio/tensorflow/README.md @@ -1,22 +1,154 @@ -# TensorFlow Lite Micro (TFLM) Architecture +# TensorFlow Lite Micro (TFLM) & Wake-on-Voice (WoV) Architecture -This directory acts as the bridge for running ML models. +This directory provides the TensorFlow Lite for Microcontrollers (TFLM) classification module (`TFLMCLY`) for Sound Open Firmware (SOF), including integration with MFCC feature extraction, mtrace logging, IPC host notifications, and Key Phrase Buffer (KPB) Wake-on-Voice (WoV) trigger infrastructure. + +--- ## Overview -Integrates TensorFlow Lite for Microcontrollers into the SOF audio pipeline. Evaluates pre-trained neural network topologies inline with the audio stream for tasks like wake-word, noise cancellation, or sound classification. +The TFLM module evaluates pre-trained micro speech neural network models inline within the SOF audio processing graph. It receives pre-processed audio feature tensors (e.g. 40-bin mel spectrograms from the MFCC component), runs model inference, logs keyword detections to `mtrace`, issues IPC4 notifications to the host audio driver, and signals the KPB module to drain buffered pre-keyword audio for Wake-on-Voice. + +--- + +## Architecture & Data Flow -## Architecture Diagram +### Dual-Path Wake-on-Voice (WoV) Architecture + +To allow continuous keyword evaluation without streaming audio to the host until a keyword is detected, the pipeline separates real-time keyword detection from host PCM draining via KPB: ```mermaid -graph LR - Feat[Audio Features] --> TFLM[TFLM Inference Engine] - SubGraph[FlatBuffer Model] -.-> TFLM - TFLM --> Out[Inference Labels/Scores] +graph TD + DAI[HDA Mic DAI] --> Gain[Gain Component] + Gain --> KPB[KPB Buffer Module] + + subgraph "Real-Time Detection Path (KPB Pin 1)" + KPB -- Live Audio Stream --> SRC[SRC: 48kHz -> 16kHz] + SRC --> MFCC[MFCC Feature Extractor] + MFCC -- 40-bin Mel Tensors --> TFLM[TFLM Classifier: tflmcly] + TFLM --> VSink[Virtual Sink: virtual.tflm_sink] + end + + subgraph "Host Draining Path (KPB Pin 2)" + KPB -- History Draining Stream --> Host[Host Copier: PCM Capture] + end + + TFLM -- "1. Log to mtrace (comp_info)" --> MTrace[mtrace / SOF Trace Log] + TFLM -- "2. IPC4 Host Notification" --> IPC[Host Audio Driver] + TFLM -- "3. KPB_EVENT_BEGIN_DRAINING (notifier_event)" --> KPB +``` + +--- + +## Pipeline Execution & Event Flow + +1. **Continuous Real-Time Listening**: + - Live microphone audio is captured by the HDA DAI and passed into `KPB` (`kpb.2.1`). + - KPB Output Pin 1 continuously streams audio to `SRC` (resampling 48kHz $\to$ 16kHz), `MFCC` (generating 40-bin `int8_t` mel spectrogram features), and `TFLM` (`tflmcly.1.1`). + - KPB stores the raw PCM audio continuously in its circular history buffer (e.g. 2100ms – 3000ms history). + +2. **Inference & Keyword Detection**: + - `tflm_process()` feeds 1960-byte feature tensors ($40 \text{ features} \times 49 \text{ windows}$) into the Micro Speech TFLM interpreter (`[1, 49, 40]` `int8_t` input tensor). + - Upon `TF_ProcessClassify()`, predictions are evaluated across categories: + - `0`: `silence` + - `1`: `unknown` + - `2`: `yes` (Keyword) + - `3`: `no` (Keyword) + +3. **Wake-on-Voice Trigger & Notification**: + When a keyword (`yes` or `no`) is detected with $\ge 0.70$ confidence: + - **mtrace Logging**: Logs detection with keyword label and confidence score via `comp_info()`: + `"TFLM keyword detected: yes (confidence 0.852)"` + - **Host IPC Notification**: Sends an IPC4 module notification (`SOF_IPC4_MODULE_NOTIFICATION`) to inform the host driver. + - **KPB Draining Signal**: Fires a system notification event (`NOTIFIER_ID_KPB_CLIENT_EVT`) with `KPB_EVENT_BEGIN_DRAINING`. + - **Host PCM Draining**: KPB opens Output Pin 2 to `host-copier`, draining pre-keyword history buffer audio followed by live mic audio to the host capture stream. + +--- + +## Topology v2 Integration & Usage + +### 1. Component Widget (`include/components/tflm.conf`) + +Defines `Class.Widget."tflmcly"`: +- **UUID**: `42:c6:1d:c5:e1:a2:df:48:a4:90:e2:74:8c:b6:36:3e` (`c51dc642-a2e1-48df-a490e2748cb6363e`) +- **Type**: `effect` + +### 2. Detection Pipeline Template (`include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf`) + +Instantiates the real-time detection graph: +```conf +Object.Widget { + virtual."1" { name "virtual.tflm_sink" } + src."1" { ... } + mfcc."1" { ... } + tflmcly."1" { ... } +} ``` -## Configuration and Scripts +### 3. Top-Level Topology Configuration (`sof-hda-tflm.conf`) + +Instantiates the complete HDA Mic WoV topology with dual-path KPB routing: +```conf +Object.Base.route [ + # DAI -> Gain -> KPB + { source "dai-copier.HDA.Analog.capture"; sink "gain.2.1" } + { source "gain.2.1"; sink "kpb.2.1" } + + # KPB Pin 1 -> Real-time Detection Path + { source "kpb.2.1"; sink "src.1.1" } + + # KPB Pin 2 -> Host WoV Draining Path + { source "kpb.2.1"; sink "host-copier.0.capture" } +] +``` + +--- + +## Building and Testing Topologies + +### Pre-processing and Compiling with `alsatplg` + +Build `sof-hda-tflm.tplg` using the Topology v2 pre-processor: + +```bash +ALSA_CONFIG_DIR=tools/topology/topology2 \ + tools/bin/alsatplg \ + -I tools/topology/topology2/ \ + -p -c tools/topology/topology2/sof-hda-tflm.conf \ + -o build/sof-hda-tflm.tplg +``` + +### Inspecting Decoded Topology Graphs + +Decode and verify the compiled `.tplg` binary: + +```bash +tools/bin/alsatplg -d build/sof-hda-tflm.tplg -o decoded.txt +grep -A 20 "SectionGraph" decoded.txt +``` + +Expected graph output: +```text +SectionGraph { + set0 { + gain.2.1 <- dai-copier.HDA.Analog.capture + kpb.2.1 <- gain.2.1 + src.1.1 <- kpb.2.1 (Real-Time Detection Path) + host-copier.0.capture <- kpb.2.1 (Host WoV Draining Path) + } + set1 { + mfcc.1.1 <- src.1.1 + tflmcly.1.1 <- mfcc.1.1 + virtual.tflm_sink <- tflmcly.1.1 (Detection Sink Termination) + } +} +``` + +--- + +## Source Files -- **Kconfig**: Enforces requirements for C++17 support and core framework staging logic (`COMP_TENSORFLOW`). -- **CMakeLists.txt**: An intricate build specification linking the Tensilica neural network library block computations (`nn_hifi_lib`) and the TensorFlow Lite micro core engine (`tflm_lib`). Also hooks the `tflm-classify.c` SOF adapter via compiler flags explicitly enforcing memory, precision, and XTENSA optimizations. -- **tflmcly.toml**: Topology definition for the specific TFLM Classifier implementation binding the engine against the UUID `UUIDREG_STR_TFLMCLY`. +- **[tflm-classify.c](file:///home/lrg/work/sof-ptl/sof/src/audio/tensorflow/tflm-classify.c)**: SOF module adapter implementation for TFLM. +- **[speech.cc](file:///home/lrg/work/sof-ptl/sof/src/audio/tensorflow/speech.cc)** / **[speech.h](file:///home/lrg/work/sof-ptl/sof/src/audio/tensorflow/speech.h)**: TFLM C++ API bridge & micro speech tensor wrapper. +- **[tflm.conf](file:///home/lrg/work/sof-ptl/sof/tools/topology/topology2/include/components/tflm.conf)**: Topology v2 widget class definition. +- **[host-gateway-src-mfcc-tflm-capture.conf](file:///home/lrg/work/sof-ptl/sof/tools/topology/topology2/include/pipelines/cavs/host-gateway-src-mfcc-tflm-capture.conf)**: Detection pipeline template. +- **[sof-hda-tflm.conf](file:///home/lrg/work/sof-ptl/sof/tools/topology/topology2/sof-hda-tflm.conf)**: Top-level WoV topology configuration. From 930de947c443c31c4913fe2435ece1bcc8093f27 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 23 Jul 2026 15:03:10 +0100 Subject: [PATCH 06/14] ptl: app: enable KPB, TFLM, MFCC, Gain, Volume components and update library base address --- app/boards/intel_adsp_ace30_ptl.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index b6ac41938398..b514c5e1e756 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -15,6 +15,12 @@ CONFIG_FORMAT_CONVERT_HIFI3=n CONFIG_COMP_GOOGLE_RTC_AUDIO_PROCESSING=m CONFIG_GOOGLE_RTC_AUDIO_PROCESSING_MOCK=y CONFIG_COMP_STFT_PROCESS=y +CONFIG_SOF_STAGING=y +CONFIG_COMP_KPB=y +CONFIG_COMP_TENSORFLOW=y +CONFIG_COMP_MFCC=y +CONFIG_COMP_VOLUME=y +CONFIG_COMP_GAIN=y # SOF / infrastructure CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n @@ -29,7 +35,7 @@ CONFIG_COLD_STORE_EXECUTE_DRAM=y CONFIG_INTEL_MODULES=y CONFIG_LIBRARY_AUTH_SUPPORT=y CONFIG_LIBRARY_MANAGER=y -CONFIG_LIBRARY_BASE_ADDRESS=0xa0688000 +CONFIG_LIBRARY_BASE_ADDRESS=0xa0700000 CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y From ea7449bf942711d4495460baea32cbe0b5ed4e26 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 23 Jul 2026 15:03:12 +0100 Subject: [PATCH 07/14] ptl: rimage: include KPB, TFLM, Volume and MFCC in base firmware manifest --- tools/rimage/config/ptl.toml.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/rimage/config/ptl.toml.h b/tools/rimage/config/ptl.toml.h index e7a53d1b1820..dbd843349cb9 100644 --- a/tools/rimage/config/ptl.toml.h +++ b/tools/rimage/config/ptl.toml.h @@ -50,9 +50,7 @@ index = __COUNTER__ #include