Speech to speech: Mute functionality

CHANGELOG.md

@@ -151,6 +151,7 @@ Breaking changes in this release:
 - 👷🏻 Added `npm run build-browser` script for building test harness package only, in PR [#5667](https://github.com/microsoft/BotFramework-WebChat/pull/5667), by [@compulim](https://github.com/compulim)
 - Added pull-based capabilities system for dynamically discovering adapter capabilities at runtime, in PR [#5679](https://github.com/microsoft/BotFramework-WebChat/pull/5679), by [@pranavjoshi001](https://github.com/pranavjoshi001)
 - Added Speech-to-Speech (S2S) support for real-time voice conversations, in PR [#5654](https://github.com/microsoft/BotFramework-WebChat/pull/5654), by [@pranavjoshi](https://github.com/pranavjoshi001)
+- Added mute/unmute functionality for speech-to-speech with silent chunks to keep server connection alive, in PR [#5688](https://github.com/microsoft/BotFramework-WebChat/pull/5688), by [@pranavjoshi](https://github.com/pranavjoshi001)
 
 ### Changed
 

__tests__/html2/speechToSpeech/barge.in.html

@@ -21,7 +21,10 @@
       3. User barges in (server detects) → "Listening..." (user speaking)
       4. Server processes → "Processing..."
       5. Bot responds with new audio → "Talk to interrupt..." (bot speaking again)
-      6. User toggles mic off
+      6. User clicks dismiss button to stop voice session
+
+      Note: Mic button toggles between listening/muted states.
+      Use dismiss button to completely stop the voice session.
     -->
     <script type="module">
       import { setupMockMediaDevices } from '/assets/esm/speechToSpeech/mockMediaDevices.js';
@@ -42,6 +45,8 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         render(
           <FluentThemeProvider variant="fluent">
@@ -183,8 +188,10 @@
         expect(activities[0]).toHaveProperty('textContent', 'Stop! Change my destination.');
         expect(activities[1]).toHaveProperty('textContent', 'Sure, where would you like to go instead?');
 
-        // Toggle mic off
-        await host.click(micButton);
+        // Click dismiss button to stop voice session
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
 
         await pageConditions.became(
           'Recording stopped',

__tests__/html2/speechToSpeech/basic.sendbox.with.mic.html

@@ -25,6 +25,8 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         render(
           <FluentThemeProvider variant="fluent">

__tests__/html2/speechToSpeech/csp.recording.html

@@ -64,6 +64,8 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         render(
           <FluentThemeProvider variant="fluent">
@@ -108,8 +110,10 @@
           1000
         );
 
-        // WHEN: User stops recording
-        await host.click(micButton);
+        // WHEN: User stops voice session using dismiss button
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
 
         // THEN: Button should change to not-recording state
         await pageConditions.became(

__tests__/html2/speechToSpeech/dtmf.input.html

@@ -37,6 +37,8 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         // Intercept postActivity to capture outgoing DTMF events
         const capturedDtmfEvents = [];
@@ -188,7 +190,10 @@
         await pageConditions.scrollToBottomCompleted();
         await host.snapshot('local');
 
-        await host.click(micButton);
+        // Stop voice recording using dismiss button
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
 
       });
     </script>

__tests__/html2/speechToSpeech/happy.path.html

@@ -32,12 +32,18 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         render(
           <FluentThemeProvider variant="fluent">
             <ReactWebChat 
               directLine={directLine}
               store={store}
+              styleOptions={{
+                disableFileUpload: true,
+                hideTelephoneKeypadButton: false,
+              }}
             />
           </FluentThemeProvider>,
           document.getElementById('webchat')
@@ -152,8 +158,10 @@
         expect(botActivityStatus.innerText).toContain('|');
         expect(botActivityStatus.innerText).toContain('Just now');
 
-        // WHEN: User stops recording by clicking microphone button again
-        await host.click(micButton);
+        // WHEN: User stops voice recording by clicking dismiss button
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
 
         // THEN: Button should change to not-recording state
         await pageConditions.became(

__tests__/html2/speechToSpeech/multiple.turns.html

@@ -41,12 +41,18 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         render(
           <FluentThemeProvider variant="fluent">
             <ReactWebChat 
               directLine={directLine}
               store={store}
+              styleOptions={{
+                disableFileUpload: true,
+                hideTelephoneKeypadButton: false,
+              }}
             />
           </FluentThemeProvider>,
           document.getElementById('webchat')
@@ -313,8 +319,10 @@
         expect(activities[4]).toHaveProperty('textContent', 'Thank you!');
         expect(activities[5]).toHaveProperty('textContent', "You're welcome! Have a safe flight.");
 
-        // ===== END: Turn off mic =====
-        await host.click(micButton);
+        // ===== END: Stop voice recording using dismiss button =====
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
 
         await pageConditions.became(
           'Recording stopped',

__tests__/html2/speechToSpeech/mute.functionality.html

@@ -0,0 +1,173 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <script crossorigin="anonymous" src="https://unpkg.com/@babel/[email protected]/babel.min.js"></script>
+    <script crossorigin="anonymous" src="https://unpkg.com/[email protected]/umd/react.production.min.js"></script>
+    <script crossorigin="anonymous" src="https://unpkg.com/[email protected]/umd/react-dom.production.min.js"></script>
+    <script crossorigin="anonymous" src="/test-harness.js"></script>
+    <script crossorigin="anonymous" src="/test-page-object.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/webchat-es5.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/botframework-webchat-fluent-theme.production.min.js"></script>
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script type="module">
+      import { setupMockMediaDevices } from '/assets/esm/speechToSpeech/mockMediaDevices.js';
+      import { setupMockAudioPlayback } from '/assets/esm/speechToSpeech/mockAudioPlayback.js';
+
+      setupMockMediaDevices();
+      setupMockAudioPlayback();
+    </script>
+    <script type="text/babel">
+      run(async function () {
+        const {
+          React,
+          ReactDOM: { render },
+          WebChat: { FluentThemeProvider, ReactWebChat, testIds }
+        } = window;
+
+        // Track voice activities sent
+        const sentVoiceActivities = [];
+
+        // GIVEN: Web Chat with Speech-to-Speech enabled
+        const { directLine, store } = testHelpers.createDirectLineEmulator();
+
+        // Set voice configuration capability to enable microphone button
+        directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
+
+        // Intercept outgoing activities to track audio chunks
+        const originalPostActivity = directLine.postActivity;
+        directLine.postActivity = function(activity) {
+          sentVoiceActivities.push(activity);
+          return originalPostActivity.call(this, activity);
+        };
+
+        render(
+          <FluentThemeProvider variant="fluent">
+            <ReactWebChat 
+              directLine={directLine}
+              store={store}
+              styleOptions={{
+                disableFileUpload: true,
+                hideTelephoneKeypadButton: false,
+              }}
+            />
+          </FluentThemeProvider>,
+          document.getElementById('webchat')
+        );
+
+        await pageConditions.uiConnected();
+
+        const micButton = document.querySelector(`[data-testid="${testIds.sendBoxMicrophoneButton}"]`);
+        expect(micButton).toBeTruthy();
+
+        // STEP 1: Click mic button to start voice session (idle → listening)
+        await host.click(micButton);
+
+        // THEN: Voice state should be 'listening'
+        await pageConditions.became(
+          'Voice state is listening',
+          () => store.getState().voice?.voiceState === 'listening',
+          1000
+        );
+
+        // THEN: Button should show recording state
+        await pageConditions.became(
+          'Microphone button changes to recording state',
+          () => {
+            const label = micButton.getAttribute('aria-label');
+            return label && (label.includes('Microphone on'));
+          },
+          1000
+        );
+
+        // Wait a bit for audio chunks to be sent
+        await new Promise(resolve => setTimeout(resolve, 200));
+
+        // Record how many chunks were sent before muting
+        const chunksBeforeMute = sentVoiceActivities.filter(
+          a => a.type === 'event' && a.name === 'media.chunk'
+        ).length;
+
+        // STEP 2: Click mic button to mute (listening → muted)
+        await host.click(micButton);
+
+        // THEN: Voice state should be 'muted'
+        await pageConditions.became(
+          'Voice state is muted',
+          () => store.getState().voice?.voiceState === 'muted',
+          1000
+        );
+
+        // Record chunks count at mute time
+        const chunksAtMute = sentVoiceActivities.filter(
+          a => a.type === 'event' && a.name === 'media.chunk'
+        ).length;
+
+        // Wait a bit - silent chunks should still be sent while muted
+        await new Promise(resolve => setTimeout(resolve, 300));
+
+        const chunksWhileMuted = sentVoiceActivities.filter(
+          a => a.type === 'event' && a.name === 'media.chunk'
+        ).length;
+
+        // THEN: Silent chunks should still be sent while muted (to keep connection alive)
+        expect(chunksWhileMuted).toBeGreaterThan(chunksAtMute);
+
+        // Take snapshot while muted
+        await host.snapshot('local');
+
+        // STEP 3: Click mic button to unmute (muted → listening)
+        await host.click(micButton);
+
+        // THEN: Voice state should be 'listening' again
+        await pageConditions.became(
+          'Voice state is listening after unmute',
+          () => store.getState().voice?.voiceState === 'listening',
+          1000
+        );
+
+        // Wait a bit for audio chunks to start flowing again
+        await new Promise(resolve => setTimeout(resolve, 200));
+
+        const chunksAfterUnmute = sentVoiceActivities.filter(
+          a => a.type === 'event' && a.name === 'media.chunk'
+        ).length;
+
+        // THEN: Chunks should continue after unmute
+        expect(chunksAfterUnmute).toBeGreaterThan(chunksWhileMuted);
+
+        // STEP 4: Click dismiss button to stop voice session
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
+
+        // THEN: Voice state should be 'idle'
+        await pageConditions.became(
+          'Voice state is idle after dismiss',
+          () => store.getState().voice?.voiceState === 'idle',
+          1000
+        );
+
+        // THEN: Button should show not-recording state
+        await pageConditions.became(
+          'Microphone button changes to not-recording state',
+          () => {
+            const label = micButton.getAttribute('aria-label');
+            return label && (label.includes('Microphone off'));
+          },
+          1000
+        );
+
+        // Verify silent chunks were sent while muted (connection kept alive)
+        expect(chunksWhileMuted).toBeGreaterThan(chunksAtMute);
+        // Verify chunks continued after unmute
+        expect(chunksAfterUnmute).toBeGreaterThan(chunksWhileMuted);
+
+      });
+    </script>
+  </body>
+</html>

__tests__/html2/speechToSpeech/outgoing.audio.interval.html

@@ -43,6 +43,8 @@
 
         // Set voice configuration capability to enable microphone button
         directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+        // Enable voice-only mode (hides send button, shows mic + dismiss buttons)
+        directLine.setCapability('getIsVoiceOnlyMode', true, { emitEvent: false });
 
         // Intercept postActivity to capture outgoing voice chunks
         const capturedChunks = [];
@@ -98,8 +100,10 @@
           2000
         );
 
-        // ===== STEP 3: Stop recording =====
-        await host.click(micButton);
+        // ===== STEP 3: Stop voice recording using dismiss button =====
+        const dismissButton = document.querySelector(`[data-testid="${testIds.sendBoxDismissButton}"]`);
+        expect(dismissButton).toBeTruthy();
+        await host.click(dismissButton);
 
         await pageConditions.became(
           'Recording stopped',

packages/api/src/boot/internal.ts

@@ -2,5 +2,6 @@ export { default as LowPriorityDecoratorComposer } from '../decorator/internal/L
 export { default as usePostVoiceActivity } from '../hooks/internal/usePostVoiceActivity';
 export { default as useSetDictateState } from '../hooks/internal/useSetDictateState';
 export { default as useShouldShowMicrophoneButton } from '../hooks/internal/useShouldShowMicrophoneButton';
+export { default as useVoiceStateWritable } from '../hooks/internal/useVoiceStateWritable';
 export { LegacyActivityContextProvider, type LegacyActivityContextType } from '../legacy/LegacyActivityBridgeContext';
 export { default as StyleOptionsComposer } from '../providers/StyleOptions/StyleOptionsComposer';

packages/api/src/localization/en-US.json

@@ -123,6 +123,8 @@
   "_SPEECH_INPUT_MICROPHONE_BUTTON_CLOSE_ALT.comment": "This is for screen reader and is the label of the microphone button, when clicked, will close microphone.",
   "SPEECH_INPUT_MICROPHONE_BUTTON_OPEN_ALT": "Microphone on",
   "_SPEECH_INPUT_MICROPHONE_BUTTON_OPEN_ALT.comment": "This is for screen reader and is the label of the microphone button, when clicked, will open microphone.",
+  "SPEECH_INPUT_STOP_RECORDING_ALT": "Stop recording",
+  "_SPEECH_INPUT_STOP_RECORDING_ALT.comment": "This is for screen reader and is the label of the dismiss button that stops recording in voice only mode.",
   "SPEECH_INPUT_STARTING": "Starting…",
   "SUGGESTED_ACTIONS_FLIPPER_NEXT_ALT": "Next",
   "_SUGGESTED_ACTIONS_FLIPPER_NEXT_ALT.comment": "This is for screen reader for the label of the right flipper button for suggested actions. Probably can re-use the value from CAROUSEL_FLIPPER_NEXT_ALT.",
@@ -133,10 +135,12 @@
   "TEXT_INPUT_ALT": "Message input box",
   "_TEXT_INPUT_ALT.comment": "This is for screen reader for the label of the message input box.",
   "TEXT_INPUT_PLACEHOLDER": "Type your message",
-  "TEXT_INPUT_SPEECH_IDLE_PLACEHOLDER": "Start talking...",
+  "TEXT_INPUT_SPEECH_IDLE_PLACEHOLDER": "Click mic to start",
   "_TEXT_INPUT_SPEECH_IDLE_PLACEHOLDER.comment": "This is the placeholder text shown in the message input box when speech-to-speech is enabled and in idle state.",
   "TEXT_INPUT_SPEECH_LISTENING_PLACEHOLDER": "Listening...",
   "_TEXT_INPUT_SPEECH_LISTENING_PLACEHOLDER.comment": "This is the placeholder text shown in the message input box when speech-to-speech is enabled and actively listening to user speech.",
+  "TEXT_INPUT_SPEECH_MUTED_PLACEHOLDER": "Muted",
+  "_TEXT_INPUT_SPEECH_MUTED_PLACEHOLDER.comment": "This is the placeholder text shown in the message input box when speech-to-speech is enabled and the microphone is muted.",
   "TEXT_INPUT_SPEECH_PROCESSING_PLACEHOLDER": "Processing...",
   "_TEXT_INPUT_SPEECH_PROCESSING_PLACEHOLDER.comment": "This is the placeholder text shown in the message input box when speech-to-speech is enabled and processing the user's speech input.",
   "TEXT_INPUT_SPEECH_BOT_SPEAKING_PLACEHOLDER": "Talk to interrupt...",

packages/api/src/providers/Capabilities/private/capabilityRegistry.ts

@@ -39,6 +39,10 @@ const CAPABILITY_REGISTRY: readonly CapabilityDescriptor<keyof Capabilities>[] =
   {
     key: 'voiceConfiguration',
     getterName: 'getVoiceConfiguration'
+  },
+  {
+    key: 'isVoiceOnlyMode',
+    getterName: 'getIsVoiceOnlyMode'
   }
 ]);