BotFramework-WebChat/__tests__/html2/speechToSpeech/happy.path.html at d916b23e02a71dc1ad3be65704a86568f72b20be · microsoft/BotFramework-WebChat · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
<!doctype html>
<html lang="en-US">
  <head>
    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
    <script crossorigin="anonymous" src="https://unpkg.com/@babel/standalone@7.8.7/babel.min.js"></script>
    <script crossorigin="anonymous" src="https://unpkg.com/react@16.8.6/umd/react.production.min.js"></script>
    <script crossorigin="anonymous" src="https://unpkg.com/react-dom@16.8.6/umd/react-dom.production.min.js"></script>
    <script crossorigin="anonymous" src="/test-harness.js"></script>
    <script crossorigin="anonymous" src="/test-page-object.js"></script>
    <script crossorigin="anonymous" src="/__dist__/webchat-es5.js"></script>
    <script crossorigin="anonymous" src="/__dist__/botframework-webchat-fluent-theme.production.min.js"></script>
  </head>
  <body>
    <main id="webchat"></main>
    <script type="module">
      import { setupMockMediaDevices } from '/assets/esm/speechToSpeech/mockMediaDevices.js';
      import { setupMockAudioPlayback } from '/assets/esm/speechToSpeech/mockAudioPlayback.js';

      setupMockMediaDevices();
      setupMockAudioPlayback();
    </script>
    <script type="text/babel">
      run(async function () {
        const {
          React,
          ReactDOM: { render },
          WebChat: { FluentThemeProvider, ReactWebChat, testIds }
        } = window;

        // GIVEN: Web Chat with Speech-to-Speech enabled
        const { directLine, store } = testHelpers.createDirectLineEmulator();

        // Server announces audio modality (mic shows up) and the consumer opted into the
        // multi-modal experience: outgoing activities go over WebSocket without echo back.
        directLine.setCapability('getVoiceConfiguration', { sampleRate: 24000, chunkIntervalMs: 100 }, { emitEvent: false });
        directLine.setCapability('getIsVoiceModeEnabled', true, { emitEvent: false });

        render(
          <FluentThemeProvider variant="fluent">
            <ReactWebChat
              directLine={directLine}
              store={store}
            />
          </FluentThemeProvider>,
          document.getElementById('webchat')
        );

        await pageConditions.uiConnected();

        const micButton = document.querySelector(`[data-testid="${testIds.sendBoxMicrophoneButton}"]`);
        expect(micButton).toBeTruthy();

        // WHEN: User clicks microphone button to start recording
        await host.click(micButton);

        // THEN: Button should show recording state
        await pageConditions.became(
          'Microphone button changes to recording state',
          () => {
            const label = micButton.getAttribute('aria-label');
            return label && (label.includes('Microphone on'));
          },
          1000
        );

        // THEN: Verify voice state is recording/listening
        await pageConditions.became(
          'Voice state is listening',
          () => store.getState().voice?.voiceState === 'listening',
          1000
        );

        // WHEN: Server detects speech start (barge-in)
        await directLine.emulateIncomingVoiceActivity({
          type: 'event',
          name: 'request.update',
          value: { state: 'detected', message: 'Your request is identified' },
          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.state'
        });


        // WHEN: Server detects speech stop (processing)
        await directLine.emulateIncomingVoiceActivity({
          type: 'event',
          name: 'request.update',
          value: { state: 'processing', message: 'Your request is being processed' },
          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.state'
        });

        // WHEN: Server sends user transcript (this goes to activities with text)
        await directLine.emulateIncomingVoiceActivity({
          type: 'event',
          name: 'media.end',
          value: { transcription: 'What is the weather today?', origin: 'user' },
          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
        });

        // THEN: User transcript appears in chat
        await pageConditions.numActivitiesShown(1);
        expect(pageElements.activityContents()[0]).toHaveProperty(
          'textContent',
          'What is the weather today?'
        );

        // THEN: Should show user message
        await pageConditions.scrollToBottomCompleted();
        await host.snapshot('local');

        // WHEN: Server sends audio chunks
        await directLine.emulateIncomingVoiceActivity({
          type: 'event',
          name: 'media.chunk',
          value: { content: 'AAAAAA==', contentType: 'audio/webm' },
          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
        });

        await directLine.emulateIncomingVoiceActivity({
          type: 'event',
          name: 'media.chunk',
          from: { role: 'bot' },
          value: { content: 'AAAAAA==', contentType: 'audio/webm' },
          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
        });


        // WHEN: Server sends bot transcript (this goes to activities with text)
        await directLine.emulateIncomingVoiceActivity({
          type: 'event',
          name: 'media.end',
          value: { transcription: 'The weather today is sunny with a high of 75 degrees.', origin: 'agent' },
          valueType: 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
        });


        // THEN: Bot transcript appears in chat
        await pageConditions.numActivitiesShown(2);

        const activities = pageElements.activityContents();
        expect(activities[0]).toHaveProperty('textContent', 'What is the weather today?');
        expect(activities[1]).toHaveProperty('textContent', 'The weather today is sunny with a high of 75 degrees.');

        // THEN: Verify activity status for voice transcripts.
        // New design: every transcript renders just `Just now | <icon>` — no role label,
        // bot uses the audio-playing icon, user uses the microphone icon.
        const activityStatuses = pageElements.activityStatuses();
        expect(activityStatuses.length).toBe(2);

        const userActivityStatus = activityStatuses[0];
        expect(userActivityStatus.innerText).toContain('Just now');
        expect(userActivityStatus.innerText).toContain('|');
        expect(userActivityStatus.querySelector('[class*="icon--microphone"]')).toBeTruthy();

        const botActivityStatus = activityStatuses[1];
        expect(botActivityStatus.innerText).toContain('Just now');
        expect(botActivityStatus.innerText).toContain('|');
        expect(botActivityStatus.querySelector('[class*="icon--audio-playing"]')).toBeTruthy();

        // WHEN: User stops recording by clicking microphone button again
        await host.click(micButton);

        // THEN: Button should change to not-recording state
        await pageConditions.became(
          'Microphone button changes to not-recording state',
          () => {
            const label = micButton.getAttribute('aria-label');
            return label && (label.includes('Microphone off'));
          },
          1000
        );

        // THEN: Should show happy path conversation
        await pageConditions.scrollToBottomCompleted();
        await host.snapshot('local');

      });
    </script>
  </body>
</html>