Core speech to speech implementation

CHANGELOG.md

@@ -150,6 +150,7 @@ Breaking changes in this release:
 - The `botframework-webchat` package now uses CSS modules for styling purposes, in PR [#5666](https://github.com/microsoft/BotFramework-WebChat/pull/5666), in PR [#5677](https://github.com/microsoft/BotFramework-WebChat/pull/5677) by [@OEvgeny](https://github.com/OEvgeny)
 - 👷🏻 Added `npm run build-browser` script for building test harness package only, in PR [#5667](https://github.com/microsoft/BotFramework-WebChat/pull/5667), by [@compulim](https://github.com/compulim)
 - Added pull-based capabilities system for dynamically discovering adapter capabilities at runtime, in PR [#5679](https://github.com/microsoft/BotFramework-WebChat/pull/5679), by [@pranavjoshi001](https://github.com/pranavjoshi001)
+- Added Speech-to-Speech (S2S) support for real-time voice conversations, in PR [#5654](https://github.com/microsoft/BotFramework-WebChat/pull/5654), by [@pranavjoshi](https://github.com/pranavjoshi001)
 
 ### Changed
 

__tests__/assets/esm/speechToSpeech/mockAudioPlayback.js

@@ -0,0 +1,23 @@
+/* global AudioContext */
+
+/**
+ * Mocks AudioContext.createBuffer to return buffers with minimum duration.
+ *
+ */
+export function setupMockAudioPlayback() {
+  const originalCreateBuffer = AudioContext.prototype.createBuffer;
+
+  AudioContext.prototype.createBuffer = function (numberOfChannels, length, sampleRate) {
+    // Ensure minimum duration of 0.5 seconds for testing
+    const minSamples = Math.floor(sampleRate * 0.5);
+    const actualLength = Math.max(length, minSamples);
+
+    return originalCreateBuffer.call(this, numberOfChannels, actualLength, sampleRate);
+  };
+
+  return {
+    restore: () => {
+      AudioContext.prototype.createBuffer = originalCreateBuffer;
+    }
+  };
+}

__tests__/assets/esm/speechToSpeech/mockMediaDevices.js

@@ -0,0 +1,87 @@
+/* global clearInterval, MessageChannel, navigator, setInterval, URL, window */
+
+/**
+ * Mocks browser audio APIs for speechToSpeech testing.
+ *
+ * - Intercepts AudioContext.audioWorklet.addModule() to prevent blob execution
+ * - Mocks AudioWorkletNode for the 'audio-recorder' processor
+ * - Mocks navigator.mediaDevices.getUserMedia() to return a test audio stream
+ */
+export function setupMockMediaDevices() {
+  const OriginalAudioContext = window.AudioContext;
+
+  // Intercept AudioContext to mock audioWorklet.addModule
+  window.AudioContext = function (options) {
+    const ctx = new OriginalAudioContext(options);
+
+    ctx.audioWorklet.addModule = url => {
+      if (url.startsWith('blob:')) {
+        URL.revokeObjectURL(url);
+      }
+      return Promise.resolve();
+    };
+
+    return ctx;
+  };
+
+  Object.setPrototypeOf(window.AudioContext, OriginalAudioContext);
+  window.AudioContext.prototype = OriginalAudioContext.prototype;
+
+  // Mock AudioWorkletNode - uses GainNode as base so source.connect() works
+  window.AudioWorkletNode = function (context, name, options) {
+    const node = context.createGain();
+    const channel = new MessageChannel();
+    let recording = false;
+    let intervalId = null;
+
+    node.port = channel.port1;
+
+    // port1 is exposed as worklet.port to the real code
+    // Real code sends to port1 → received by port2.onmessage (commands)
+    // Mock sends from port2 → received by port1.onmessage (audio chunks)
+    channel.port2.onmessage = ({ data }) => {
+      if (data.command === 'START') {
+        recording = true;
+        const bufferSize = options?.processorOptions?.bufferSize || 2400;
+
+        // Send chunks at ~100ms intervals while recording
+        // Use port2.postMessage so port1.onmessage (set by real code) receives it
+        intervalId = setInterval(() => {
+          if (recording) {
+            channel.port2.postMessage({ eventType: 'audio', audioData: new Float32Array(bufferSize) });
+          }
+        }, 100);
+      } else if (data.command === 'STOP') {
+        recording = false;
+        if (intervalId) {
+          clearInterval(intervalId);
+          intervalId = null;
+        }
+      }
+    };
+
+    return node;
+  };
+
+  // Mock getUserMedia with oscillator-based test stream
+  navigator.mediaDevices.getUserMedia = constraints => {
+    const sampleRate = constraints?.audio?.sampleRate || 24000;
+    const ctx = new OriginalAudioContext({ sampleRate });
+    const oscillator = ctx.createOscillator();
+    const destination = ctx.createMediaStreamDestination();
+
+    oscillator.connect(destination);
+    oscillator.start();
+
+    destination.stream.getTracks().forEach(track => {
+      const originalStop = track.stop.bind(track);
+      track.stop = () => {
+        oscillator.stop();
+        ctx.close();
+        originalStop();
+      };
+    });
+
+    return Promise.resolve(destination.stream);
+  };
+}

__tests__/html2/speechToSpeech/barge.in.html

@@ -0,0 +1,197 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <script crossorigin="anonymous" src="https://unpkg.com/@babel/standalone@7.8.7/babel.min.js"></script>
+    <script crossorigin="anonymous" src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
+    <script crossorigin="anonymous" src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
+    <script crossorigin="anonymous" src="/test-harness.js"></script>
+    <script crossorigin="anonymous" src="/test-page-object.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/webchat-es5.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/botframework-webchat-fluent-theme.production.min.js"></script>
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <!--
+      Test: Barge-in scenario with full state cycle
+
+      Flow:
+      1. User starts recording → "Listening..."
+      2. Bot sends audio chunks → "Talk to interrupt..." (bot speaking)
+      3. User barges in (server detects) → "Listening..." (user speaking)
+      4. Server processes → "Processing..."
+      5. Bot responds with new audio → "Talk to interrupt..." (bot speaking again)
+      6. User toggles mic off
+    -->
+    <script type="module">
+      import { setupMockMediaDevices } from '/assets/esm/speechToSpeech/mockMediaDevices.js';
+      import { setupMockAudioPlayback } from '/assets/esm/speechToSpeech/mockAudioPlayback.js';
+
+      setupMockMediaDevices();
+      setupMockAudioPlayback();
+    </script>
+    <script type="text/babel">
+      run(async function () {
+        const {
+          React,
+          ReactDOM: { render },
+          WebChat: { FluentThemeProvider, ReactWebChat, testIds }
+        } = window;
+
+        const { directLine, store } = testHelpers.createDirectLineEmulator();
+
+        // Set voice configuration capability to enable microphone button
+        directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+
+        render(
+          <FluentThemeProvider variant="fluent">
+            <ReactWebChat 
+              directLine={directLine}
+              store={store}
+            />
+          </FluentThemeProvider>,
+          document.getElementById('webchat')
+        );
+
+        await pageConditions.uiConnected();
+
+        const micButton = document.querySelector(`[data-testid="${testIds.sendBoxMicrophoneButton}"]`);
+        const textArea = document.querySelector(`[data-testid="${testIds.sendBoxTextBox}"]`);
+        expect(micButton).toBeTruthy();
+        expect(textArea).toBeTruthy();
+
+        // Start recording
+        await host.click(micButton);
+
+        await pageConditions.became(
+          'Recording started',
+          () => micButton.getAttribute('aria-label')?.includes('Microphone on'),
+          1000
+        );
+
+        // VERIFY: State is "listening"
+        await pageConditions.became(
+          'State: listening → Placeholder: "Listening..."',
+          () => textArea.getAttribute('placeholder') === 'Listening...',
+          2000
+        );
+
+        // Bot starts speaking (sends audio chunks)
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'media.chunk',
+          from: { role: 'bot' },
+          value: { content: 'AAAAAA==', contentType: 'audio/webm' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+        });
+
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'media.chunk',
+          from: { role: 'bot' },
+          value: { content: 'AAAAAA==', contentType: 'audio/webm' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+        });
+
+        // VERIFY: State is "bot_speaking" (isPlaying = true)
+        await pageConditions.became(
+          'State: bot_speaking → Placeholder: "Talk to interrupt..."',
+          () => textArea.getAttribute('placeholder') === 'Talk to interrupt...',
+          1000
+        );
+
+        // VERIFY: Mic button has pulse animation during bot speaking
+        expect(micButton.className).toMatch(/with-pulse/);
+
+        // User barges in (server detects user speech)
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'request.update',
+          from: { role: 'bot' },
+          value: { state: 'detected', message: 'Your request is identified' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.state'
+        });
+
+        // VERIFY: State changes to "user_speaking" - bot audio stopped
+        await pageConditions.became(
+          'State: user_speaking → Placeholder: "Listening…" (barge-in worked)',
+          () => textArea.getAttribute('placeholder') === 'Listening...',
+          1000
+        );
+
+        // VERIFY: Mic button still has pulse animation during user speaking
+        expect(micButton.className).toMatch(/with-pulse/);
+
+        // Server processes the user's interrupted request
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'request.update',
+          from: { role: 'bot' },
+          value: { state: 'processing', message: 'Your request is being processed' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.state'
+        });
+
+        // VERIFY: State is "processing"
+        await pageConditions.became(
+          'State: processing → Placeholder: "Processing…"',
+          () => textArea.getAttribute('placeholder') === 'Processing...',
+          1000
+        );
+
+        // User transcript arrives
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'media.end',
+          from: { role: 'bot' },
+          text: 'Stop! Change my destination.',
+          value: { transcription: 'Stop! Change my destination.', origin: 'user' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
+        });
+
+        await pageConditions.numActivitiesShown(1);
+
+        // Bot responds with new audio
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'media.chunk',
+          from: { role: 'bot' },
+          value: { content: 'AAAAAA==', contentType: 'audio/webm' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+        });
+
+        // VERIFY: State is "bot_speaking" again
+        await pageConditions.became(
+          'State: bot_speaking → Placeholder: "Talk to interrupt..." (bot responding)',
+          () => textArea.getAttribute('placeholder') === 'Talk to interrupt...',
+          1000
+        );
+
+        // Bot transcript arrives
+        await directLine.emulateIncomingVoiceActivity({
+          type: 'event',
+          name: 'media.end',
+          from: { role: 'bot' },
+          text: 'Sure, where would you like to go instead?',
+          value: { transcription: 'Sure, where would you like to go instead?', origin: 'agent' },
+          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
+        });
+
+        await pageConditions.numActivitiesShown(2);
+
+        // Verify both messages appear
+        const activities = pageElements.activityContents();
+        expect(activities[0]).toHaveProperty('textContent', 'Stop! Change my destination.');
+        expect(activities[1]).toHaveProperty('textContent', 'Sure, where would you like to go instead?');
+
+        // Toggle mic off
+        await host.click(micButton);
+
+        await pageConditions.became(
+          'Recording stopped',
+          () => micButton.getAttribute('aria-label')?.includes('Microphone off'),
+          1000
+        );
+      });
+    </script>
+  </body>
+</html>

__tests__/html2/speechToSpeech/basic.sendbox.with.mic.html

@@ -0,0 +1,80 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <script crossorigin="anonymous" src="https://unpkg.com/@babel/standalone@7.8.7/babel.min.js"></script>
+    <script crossorigin="anonymous" src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
+    <script crossorigin="anonymous" src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
+    <script crossorigin="anonymous" src="/test-harness.js"></script>
+    <script crossorigin="anonymous" src="/test-page-object.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/webchat-es5.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/botframework-webchat-fluent-theme.production.min.js"></script>
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script type="text/babel">
+      run(async function () {
+        const {
+          React,
+          ReactDOM: { render },
+          WebChat: { FluentThemeProvider, ReactWebChat, testIds }
+        } = window;
+
+        // GIVEN: Web Chat with Fluent Theme and microphone button enabled
+        const { directLine, store } = testHelpers.createDirectLineEmulator();
+
+        // Set voice configuration capability to enable microphone button
+        directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
+
+        render(
+          <FluentThemeProvider variant="fluent">
+            <ReactWebChat 
+              directLine={directLine}
+              store={store}
+              styleOptions={{
+                disableFileUpload: true,
+                hideTelephoneKeypadButton: false,
+              }}
+            />
+          </FluentThemeProvider>,
+          document.getElementById('webchat')
+        );
+
+        await pageConditions.uiConnected();
+
+        // THEN: Microphone button should be present
+        const micButton = document.querySelector(`[data-testid="${testIds.sendBoxMicrophoneButton}"]`);
+        expect(micButton).toBeTruthy();
+
+        // THEN: Telephone keypad button should be present
+        const keypadButton = document.querySelector(`[data-testid="${testIds.sendBoxTelephoneKeypadToolbarButton}"]`);
+        expect(keypadButton).toBeTruthy();
+
+        // THEN: Text counter should NOT be present
+        const textCounter = document.querySelector('.sendbox__text-counter');
+        expect(textCounter).toBeFalsy();
+
+        // THEN: Send button should NOT be present
+        const sendButton = document.querySelector(`[data-testid="${testIds.sendBoxSendButton}"]`);
+        expect(sendButton).toBeFalsy();
+
+        // THEN: Should show sendbox with microphone and keypad buttons
+        await host.snapshot('local');
+
+        // WHEN: Voice configuration is removed from directLine
+        directLine.setCapability('getVoiceConfiguration', undefined);
+
+        // Wait for UI to update
+        await pageConditions.became(
+          'Microphone button should be hidden after removing voice configuration',
+          () => !document.querySelector(`[data-testid="${testIds.sendBoxMicrophoneButton}"]`),
+          1000
+        );
+
+        // THEN: Microphone button should NOT be present anymore
+        const micButtonAfterRemoval = document.querySelector(`[data-testid="${testIds.sendBoxMicrophoneButton}"]`);
+        expect(micButtonAfterRemoval).toBeFalsy();
+      });
+    </script>
+  </body>
+</html>