QEo8ui496WQqqmM~U27_KnO;kOTfSi<;0S_nP
zYQ>cYG}nckh*jNKi6`?v>(ZvOG^6c82u=PT2o%7t*D3dZ@GuwHH$rU3jEHK?rSbZZaBqwSqITqgL0IHQg^lU+t%d!QR($RzVb7vHa5x-roEUTw(0{vG=)065*F~q^rJsFSFk4z|k2S|7SXS
zXNOHb52Mx4UiisHuEeOzH|h?=^MNX~iqm^xCIn+}+D`W&E;!b2rh$O1uS&R-g_u%+
zYfn^#Gc98HPz(nM{8cDnr)${-CFg1DK3F>OumAUdE0ax|3G!3pE5*!K=NZOv;{Y%|
LIPVUxtJMDjb8b2@
literal 0
HcmV?d00001
diff --git a/samples/csharp/web/avatar/wwwroot/image/favicon.ico b/samples/csharp/web/avatar/wwwroot/image/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..63e859b476eff5055e0e557aaa151ca8223fbeef
GIT binary patch
literal 5430
zcmc&&Yj2xp8Fqnv;>&(QB_ve7>^E#o2mu=cO~A%R>DU-_hfbSRv1t;m7zJ_AMrntN
zy0+^f&8be>q&YYzH%(88lQ?#KwiCzaCO*ZEo%j&v;<}&Lj_stKTKK>#U3nin@AF>w
zb3ONSAFR{u(S1d?cdw53y}Gt1b-Hirbh;;bm(Rcbnoc*%@jiaXM|4jU^1WO~`TYZ~
zC-~jh9~b-f?fX`DmwvcguQzn*uV}c^Vd&~?H|RUs4Epv~gTAfR(B0lT&?RWQOtduM
z^1vUD9{HQsW!{a9|0crA34m7Z6lpG^}f6f?={zD+
zXAzk^i^aKN_}s2$eX81wjSMONE#WVdzf|MT)Ap*}Vsn!XbvsI#6o&ij{87^d%$|A{
z=F{KB%)g%@z76yBzbb7seW**Ju8r4e*Z3PWNX3_tTDgzZatz7)Q6ytwB%@&@A|XT;
zecM`Snxx5po$C)%yCP!KEtos~eOS)@2=kX-RIm)4glMCoagTEFxrBeSX%Euz734Fk
z%7)x(k~T!@Hbg_37NSQL!vlTBXoURSzt~I**Zw`&F24fH*&kx=%nvZv|49SC*daD(
zIw<~%#=lk8{2-l(BcIjy^Q$Q&m#KlWL9?UG{b8@qhlD
z;umc+6p%|NsAT~0@DgV4-NKgQuWPWrmPIK&&XhV&n%`{l
zOl^bbWYjQNuVXTXESO)@|iUKVmErPUDfz2Wh`4dF@OFiaCW|d`3paV^@|r^8T_ZxM)Z+$p5qx#
z#K=z@%;aBPO=C4JNNGqVv6@UGolIz;KZsAro``Rz8X%vq_gpi^qEV&evgHb_=Y9-l
z`)imdx0UC>GWZYj)3+3aKh?zVb}=@%oNzg7a8%kfVl)SV-Amp1Okw&+hEZ3|v(k8vRjXW9?ih`&FFM
zV$~{j3IzhtcXk?Mu_!12;=+I7XK-IR2>Yd%VB^?oI9c^E&Chb&&je$NV0P-R;ujkP
z;cbLCCPEF6|22NDj=S`F^2e~XwT1ZnRX8ra0#DaFa9-X|8(xNW_+JhD75WnSd7cxo
z2>I_J5{c|WPfrgl7E2R)^c}F7ry()Z>$Jhk9CzZxiPKL#_0%`&{MX>P_%b~Dx0D^S
z7xP1(DQ!d_Icpk!RN3I1w@~|O1ru#CO==h#9M~S4Chx*@?=EKUPGBv$tmU+7Zs_al
z`!jR?6T&Z7(%uVq>#yLu`abWk!FBlnY{RFNHlj~6zh*;@u}+}viRKsD`IIxN#R-X3
z@vxu#EA_m}I503U(8Qmx^}u;)KfGP`O9E1H1Q|xeeksX8jC%@!{YT1)!lWgO=+Y3*jr=iSxvOW1}^HSy=y){tOMQJ@an>sOl4FYniE
z;GOxd7AqxZNbYFNqobpv&HVO$c-w!Y*6r;$2oJ~h(a#(Bp<-)dg*mNigX~9rPqcHv
z^;c*|Md?tD)$y?6FO$DWl$jUGV`F1G_^E&E>sY*YnA~ruv3=z9F8&&~Xpm<<75?N3
z>x~`I&M9q)O1=zWZHN9hZWx>RQ}zLP+iL57Q)%&_^$Sme^^G7;e-P~CR?kqU#Io#(
z(nH1Wn*Ig)|M>WLGrxoU?FZrS`4GO&w;+39A3f8w{{Q7eg|$+dIlNFPAe+tN=FOYU
z{A&Fg|H73+w1IK(W=j*L>JQgz$g0
z7JpKXLHIh}#$wm|N`s}o-@|L_`>*(gTQ~)wr3Eap7g%PVNisKw82im;Gdv#85x#s+
zoqqtnwu4ycd>cOQgRh-=aEJbnvVK`}ja%+FZx}&ehtX)n(9nVfe4{mn0bgijUbNr7Tf5X^$*{qh2%`?--%+sbSrjE^;1e3>%
zqa%jdY16{Y)a1hSy*mr0JGU05Z%=qlx5vGvTjSpTt6k%nR06q}1DU`SQh_ZAeJ}A@`hL~xvv05U?0%=spP`R>dk?cOWM9^KNb7B?xjex>OZo%JMQQ1Q
zB|q@}8RiP@DWn-(fB;phPaIOP2Yp)XN3-Fsn)S3w($4&+p8f5W_f%gac}QvmkHfCj$2=!t`boCvQ
zCW;&Dto=f8v##}dy^wg3VNaBy&kCe3N;1|@n@pUaMPT?(aJ9b*(gJ28$}(2qFt$H~u5z94xcIQkcOI++)*exzbrk?WOOOf*|%k5#KV
zL=&ky3)Eirv$wbRJ2F2s_ILQY--D~~7>^f}W|Aw^e7inXr#WLI{@h`0|jHud2Y~cI~Yn{r_kU^Vo{1gja {
+ document.getElementById('logging').innerHTML += msg + '
'
+}
+
+// Setup WebRTC
+function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
+ // Create WebRTC peer connection
+ peerConnection = new RTCPeerConnection({
+ iceServers: [{
+ urls: [ iceServerUrl ],
+ username: iceServerUsername,
+ credential: iceServerCredential
+ }],
+ iceTransportPolicy: 'relay'
+ })
+
+ // Fetch WebRTC video stream and mount it to an HTML video element
+ peerConnection.ontrack = function (event) {
+ // Clean up existing video element if there is any
+ remoteVideoDiv = document.getElementById('remoteVideo')
+ for (var i = 0; i < remoteVideoDiv.childNodes.length; i++) {
+ if (remoteVideoDiv.childNodes[i].localName === event.track.kind) {
+ remoteVideoDiv.removeChild(remoteVideoDiv.childNodes[i])
+ }
+ }
+
+ const mediaPlayer = document.createElement(event.track.kind)
+ mediaPlayer.id = event.track.kind
+ mediaPlayer.srcObject = event.streams[0]
+ mediaPlayer.autoplay = true
+ document.getElementById('remoteVideo').appendChild(mediaPlayer)
+ document.getElementById('videoLabel').hidden = true
+ document.getElementById('overlayArea').hidden = false
+
+ if (event.track.kind === 'video') {
+ mediaPlayer.playsInline = true
+ remoteVideoDiv = document.getElementById('remoteVideo')
+ canvas = document.getElementById('canvas')
+ if (document.getElementById('transparentBackground').checked) {
+ remoteVideoDiv.style.width = '0.1px'
+ canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height)
+ canvas.hidden = false
+ } else {
+ canvas.hidden = true
+ }
+
+ mediaPlayer.addEventListener('play', () => {
+ if (document.getElementById('transparentBackground').checked) {
+ window.requestAnimationFrame(makeBackgroundTransparent)
+ } else {
+ remoteVideoDiv.style.width = mediaPlayer.videoWidth / 2 + 'px'
+ }
+ })
+ }
+ else
+ {
+ // Mute the audio player to make sure it can auto play, will unmute it when speaking
+ // Refer to https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide
+ mediaPlayer.muted = true
+ }
+ }
+
+ // Listen to data channel, to get the event from the server
+ peerConnection.addEventListener("datachannel", event => {
+ const dataChannel = event.channel
+ dataChannel.onmessage = e => {
+ console.log("[" + (new Date()).toISOString() + "] WebRTC event received: " + e.data)
+
+ if (e.data.includes("EVENT_TYPE_SWITCH_TO_IDLE")) {
+ document.getElementById('speak').disabled = false
+ document.getElementById('stopSpeaking').disabled = true
+ }
+ }
+ })
+
+ // This is a workaround to make sure the data channel listening is working by creating a data channel from the client side
+ c = peerConnection.createDataChannel("eventChannel")
+
+ // Make necessary update to the web page when the connection state changes
+ peerConnection.oniceconnectionstatechange = e => {
+ log("WebRTC status: " + peerConnection.iceConnectionState)
+
+ if (peerConnection.iceConnectionState === 'connected') {
+ document.getElementById('stopSession').disabled = false
+ document.getElementById('speak').disabled = false
+ document.getElementById('configuration').hidden = true
+ }
+
+ if (peerConnection.iceConnectionState === 'disconnected' || peerConnection.iceConnectionState === 'failed') {
+ document.getElementById('speak').disabled = true
+ document.getElementById('stopSpeaking').disabled = true
+ document.getElementById('stopSession').disabled = true
+ document.getElementById('startSession').disabled = false
+ document.getElementById('configuration').hidden = false
+ }
+ }
+
+ // Offer to receive 1 audio, and 1 video track
+ peerConnection.addTransceiver('video', { direction: 'sendrecv' })
+ peerConnection.addTransceiver('audio', { direction: 'sendrecv' })
+
+ // Connect to avatar service when ICE candidates gathering is done
+ iceGatheringDone = false
+
+ peerConnection.onicecandidate = e => {
+ if (!e.candidate && !iceGatheringDone) {
+ iceGatheringDone = true
+ connectToAvatarService(peerConnection)
+ }
+ }
+
+ peerConnection.createOffer().then(sdp => {
+ peerConnection.setLocalDescription(sdp).then(() => { setTimeout(() => {
+ if (!iceGatheringDone) {
+ iceGatheringDone = true
+ connectToAvatarService(peerConnection)
+ }
+ }, 2000) })
+ })
+}
+
+// Connect to TTS Avatar Service
+function connectToAvatarService(peerConnection) {
+ let localSdp = btoa(JSON.stringify(peerConnection.localDescription))
+ let headers = {
+ 'ClientId': clientId,
+ 'AvatarCharacter': document.getElementById('talkingAvatarCharacter').value,
+ 'AvatarStyle': document.getElementById('talkingAvatarStyle').value,
+ 'BackgroundColor': document.getElementById('backgroundColor').value,
+ 'BackgroundImageUrl': document.getElementById('backgroundImageUrl').value,
+ 'IsCustomAvatar': document.getElementById('customizedAvatar').checked,
+ 'TransparentBackground': document.getElementById('transparentBackground').checked,
+ 'VideoCrop': document.getElementById('videoCrop').checked
+ }
+
+ if (document.getElementById('customVoiceEndpointId').value !== '') {
+ headers['CustomVoiceEndpointId'] = document.getElementById('customVoiceEndpointId').value
+ }
+
+ fetch('/api/connectAvatar', {
+ method: 'POST',
+ headers: headers,
+ body: localSdp
+ })
+ .then(response => {
+ if (response.ok) {
+ response.text().then(text => {
+ const remoteSdp = text
+ peerConnection.setRemoteDescription(new RTCSessionDescription(JSON.parse(atob(remoteSdp))))
+ })
+ } else {
+ document.getElementById('startSession').disabled = false;
+ document.getElementById('configuration').hidden = false;
+ throw new Error(`Failed connecting to the Avatar service: ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+// Make video background transparent by matting
+function makeBackgroundTransparent(timestamp) {
+ // Throttle the frame rate to 30 FPS to reduce CPU usage
+ if (timestamp - previousAnimationFrameTimestamp > 30) {
+ video = document.getElementById('video')
+ tmpCanvas = document.getElementById('tmpCanvas')
+ tmpCanvasContext = tmpCanvas.getContext('2d', { willReadFrequently: true })
+ tmpCanvasContext.drawImage(video, 0, 0, video.videoWidth, video.videoHeight)
+ if (video.videoWidth > 0) {
+ let frame = tmpCanvasContext.getImageData(0, 0, video.videoWidth, video.videoHeight)
+ for (let i = 0; i < frame.data.length / 4; i++) {
+ let r = frame.data[i * 4 + 0]
+ let g = frame.data[i * 4 + 1]
+ let b = frame.data[i * 4 + 2]
+ if (g - 150 > r + b) {
+ // Set alpha to 0 for pixels that are close to green
+ frame.data[i * 4 + 3] = 0
+ } else if (g + g > r + b) {
+ // Reduce green part of the green pixels to avoid green edge issue
+ adjustment = (g - (r + b) / 2) / 3
+ r += adjustment
+ g -= adjustment * 2
+ b += adjustment
+ frame.data[i * 4 + 0] = r
+ frame.data[i * 4 + 1] = g
+ frame.data[i * 4 + 2] = b
+ // Reduce alpha part for green pixels to make the edge smoother
+ a = Math.max(0, 255 - adjustment * 4)
+ frame.data[i * 4 + 3] = a
+ }
+ }
+
+ canvas = document.getElementById('canvas')
+ canvasContext = canvas.getContext('2d')
+ canvasContext.putImageData(frame, 0, 0);
+ }
+
+ previousAnimationFrameTimestamp = timestamp
+ }
+
+ window.requestAnimationFrame(makeBackgroundTransparent)
+}
+
+// Do HTML encoding on given text
+function htmlEncode(text) {
+ const entityMap = {
+ '&': '&',
+ '<': '<',
+ '>': '>',
+ '"': '"',
+ "'": ''',
+ '/': '/'
+ };
+
+ return String(text).replace(/[&<>"'\/]/g, (match) => entityMap[match])
+}
+
+window.onload = () => {
+ clientId = document.getElementById('clientId').value
+}
+
+window.startSession = () => {
+ document.getElementById('startSession').disabled = true
+
+ fetch('/api/getIceToken', {
+ method: 'GET',
+ })
+ .then(response => {
+ if (response.ok) {
+ response.json().then(data => {
+ const iceServerUrl = data.Urls[0]
+ const iceServerUsername = data.Username
+ const iceServerCredential = data.Password
+ setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential)
+ })
+ } else {
+ throw new Error(`Failed fetching ICE token: ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+window.speak = () => {
+ document.getElementById('speak').disabled = true;
+ document.getElementById('stopSpeaking').disabled = false
+ document.getElementById('audio').muted = false
+ let spokenText = document.getElementById('spokenText').value
+ let ttsVoice = document.getElementById('ttsVoice').value
+ let personalVoiceSpeakerProfileID = document.getElementById('personalVoiceSpeakerProfileID').value
+ let spokenSsml = `${htmlEncode(spokenText)}`
+ console.log("[" + (new Date()).toISOString() + "] Speak request sent.")
+
+ fetch('/api/speak', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId,
+ 'Content-Type': 'application/ssml+xml'
+ },
+ body: spokenSsml
+ })
+ .then(response => {
+ if (response.ok) {
+ response.text().then(text => {
+ console.log(`[${new Date().toISOString()}] Speech synthesized to speaker for text [ ${spokenText} ]. Result ID: ${text}`)
+ })
+ } else {
+ throw new Error(`[${new Date().toISOString()}] Unable to speak text. ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+window.stopSpeaking = () => {
+ document.getElementById('stopSpeaking').disabled = true
+
+ fetch('/api/stopSpeaking', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId
+ },
+ body: ''
+ })
+ .then(response => {
+ if (response.ok) {
+ console.log(`[${new Date().toISOString()}] Speaking stopped.`)
+ } else {
+ throw new Error(`[${new Date().toISOString()}] Unable to stop speaking. ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+window.stopSession = () => {
+ document.getElementById('speak').disabled = true
+ document.getElementById('stopSpeaking').disabled = true
+ document.getElementById('stopSession').disabled = true
+
+ fetch('/api/disconnectAvatar', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId
+ },
+ body: ''
+ })
+}
+
+window.updataTransparentBackground = () => {
+ if (document.getElementById('transparentBackground').checked) {
+ document.body.background = './image/background.png'
+ document.getElementById('backgroundColor').value = '#00FF00FF'
+ document.getElementById('backgroundColor').disabled = true
+ document.getElementById('backgroundImageUrl').value = ''
+ document.getElementById('backgroundImageUrl').disabled = true
+ } else {
+ document.body.background = ''
+ document.getElementById('backgroundColor').value = '#FFFFFFFF'
+ document.getElementById('backgroundColor').disabled = false
+ document.getElementById('backgroundImageUrl').disabled = false
+ }
+}
diff --git a/samples/csharp/web/avatar/wwwroot/js/chat.js b/samples/csharp/web/avatar/wwwroot/js/chat.js
new file mode 100644
index 000000000..7d35eacf2
--- /dev/null
+++ b/samples/csharp/web/avatar/wwwroot/js/chat.js
@@ -0,0 +1,539 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license.
+
+// Global objects
+var clientId
+var speechRecognizer
+var peerConnection
+var isSpeaking = false
+var sessionActive = false
+var lastSpeakTime
+var isFirstRecognizingEvent = true
+
+// Connect to avatar service
+function connectAvatar() {
+ document.getElementById('startSession').disabled = true
+
+ fetch('/api/getIceToken', {
+ method: 'GET',
+ })
+ .then(response => {
+ if (response.ok) {
+ response.json().then(data => {
+ const iceServerUrl = data.Urls[0]
+ const iceServerUsername = data.Username
+ const iceServerCredential = data.Password
+ setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential)
+ })
+ } else {
+ throw new Error(`Failed fetching ICE token: ${response.status} ${response.statusText}`)
+ }
+ })
+
+ document.getElementById('configuration').hidden = true
+}
+
+// Create speech recognizer
+function createSpeechRecognizer() {
+ fetch('/api/getSpeechToken', {
+ method: 'GET',
+ })
+ .then(response => {
+ if (response.ok) {
+ const speechRegion = response.headers.get('SpeechRegion')
+ const speechPrivateEndpoint = response.headers.get('SpeechPrivateEndpoint')
+ response.text().then(text => {
+ const speechToken = text
+ const speechRecognitionConfig = speechPrivateEndpoint ?
+ SpeechSDK.SpeechConfig.fromEndpoint(new URL(`wss://${speechPrivateEndpoint.replace('https://', '')}/stt/speech/universal/v2`), '') :
+ SpeechSDK.SpeechConfig.fromEndpoint(new URL(`wss://${speechRegion}.stt.speech.microsoft.com/speech/universal/v2`), '')
+ speechRecognitionConfig.authorizationToken = speechToken
+ speechRecognitionConfig.setProperty(SpeechSDK.PropertyId.SpeechServiceConnection_LanguageIdMode, "Continuous")
+ speechRecognitionConfig.setProperty("SpeechContext-PhraseDetection.TrailingSilenceTimeout", "3000")
+ speechRecognitionConfig.setProperty("SpeechContext-PhraseDetection.InitialSilenceTimeout", "10000")
+ speechRecognitionConfig.setProperty("SpeechContext-PhraseDetection.Dictation.Segmentation.Mode", "Custom")
+ speechRecognitionConfig.setProperty("SpeechContext-PhraseDetection.Dictation.Segmentation.SegmentationSilenceTimeoutMs", "200")
+ var sttLocales = document.getElementById('sttLocales').value.split(',')
+ var autoDetectSourceLanguageConfig = SpeechSDK.AutoDetectSourceLanguageConfig.fromLanguages(sttLocales)
+ speechRecognizer = SpeechSDK.SpeechRecognizer.FromConfig(speechRecognitionConfig, autoDetectSourceLanguageConfig, SpeechSDK.AudioConfig.fromDefaultMicrophoneInput())
+ })
+ } else {
+ throw new Error(`Failed fetching speech token: ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+// Disconnect from avatar service
+function disconnectAvatar(closeSpeechRecognizer = false) {
+ fetch('/api/disconnectAvatar', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId
+ },
+ body: ''
+ })
+
+ if (speechRecognizer !== undefined) {
+ speechRecognizer.stopContinuousRecognitionAsync()
+ if (closeSpeechRecognizer) {
+ speechRecognizer.close()
+ }
+ }
+
+ sessionActive = false
+}
+
+// Setup WebRTC
+function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
+ // Create WebRTC peer connection
+ peerConnection = new RTCPeerConnection({
+ iceServers: [{
+ urls: [ iceServerUrl ],
+ username: iceServerUsername,
+ credential: iceServerCredential
+ }],
+ iceTransportPolicy: 'relay'
+ })
+
+ // Fetch WebRTC video stream and mount it to an HTML video element
+ peerConnection.ontrack = function (event) {
+ if (event.track.kind === 'audio') {
+ let audioElement = document.createElement('audio')
+ audioElement.id = 'audioPlayer'
+ audioElement.srcObject = event.streams[0]
+ audioElement.autoplay = true
+
+ audioElement.onplaying = () => {
+ console.log(`WebRTC ${event.track.kind} channel connected.`)
+ }
+
+ document.getElementById('remoteVideo').appendChild(audioElement)
+ }
+
+ if (event.track.kind === 'video') {
+ let videoElement = document.createElement('video')
+ videoElement.id = 'videoPlayer'
+ videoElement.srcObject = event.streams[0]
+ videoElement.autoplay = true
+ videoElement.playsInline = true
+
+ videoElement.onplaying = () => {
+ // Clean up existing video element if there is any
+ remoteVideoDiv = document.getElementById('remoteVideo')
+ for (var i = 0; i < remoteVideoDiv.childNodes.length; i++) {
+ if (remoteVideoDiv.childNodes[i].localName === event.track.kind) {
+ remoteVideoDiv.removeChild(remoteVideoDiv.childNodes[i])
+ }
+ }
+
+ // Append the new video element
+ document.getElementById('remoteVideo').appendChild(videoElement)
+
+ console.log(`WebRTC ${event.track.kind} channel connected.`)
+ document.getElementById('microphone').disabled = false
+ document.getElementById('stopSession').disabled = false
+ document.getElementById('remoteVideo').style.width = '960px'
+ document.getElementById('chatHistory').hidden = false
+ document.getElementById('showTypeMessage').disabled = false
+
+ if (document.getElementById('useLocalVideoForIdle').checked) {
+ document.getElementById('localVideo').hidden = true
+ if (lastSpeakTime === undefined) {
+ lastSpeakTime = new Date()
+ }
+ }
+
+ setTimeout(() => { sessionActive = true }, 5000) // Set session active after 5 seconds
+ }
+ }
+ }
+
+ // Listen to data channel, to get the event from the server
+ peerConnection.addEventListener("datachannel", event => {
+ const dataChannel = event.channel
+ dataChannel.onmessage = e => {
+ console.log("[" + (new Date()).toISOString() + "] WebRTC event received: " + e.data)
+
+ if (e.data.includes("EVENT_TYPE_SWITCH_TO_SPEAKING")) {
+ isSpeaking = true
+ document.getElementById('stopSpeaking').disabled = false
+ } else if (e.data.includes("EVENT_TYPE_SWITCH_TO_IDLE")) {
+ isSpeaking = false
+ lastSpeakTime = new Date()
+ document.getElementById('stopSpeaking').disabled = true
+ }
+ }
+ })
+
+ // This is a workaround to make sure the data channel listening is working by creating a data channel from the client side
+ c = peerConnection.createDataChannel("eventChannel")
+
+ // Make necessary update to the web page when the connection state changes
+ peerConnection.oniceconnectionstatechange = e => {
+ console.log("WebRTC status: " + peerConnection.iceConnectionState)
+ if (peerConnection.iceConnectionState === 'disconnected') {
+ if (document.getElementById('useLocalVideoForIdle').checked) {
+ document.getElementById('localVideo').hidden = false
+ document.getElementById('remoteVideo').style.width = '0.1px'
+ }
+ }
+ }
+
+ // Offer to receive 1 audio, and 1 video track
+ peerConnection.addTransceiver('video', { direction: 'sendrecv' })
+ peerConnection.addTransceiver('audio', { direction: 'sendrecv' })
+
+ // Connect to avatar service when ICE candidates gathering is done
+ iceGatheringDone = false
+
+ peerConnection.onicecandidate = e => {
+ if (!e.candidate && !iceGatheringDone) {
+ iceGatheringDone = true
+ connectToAvatarService(peerConnection)
+ }
+ }
+
+ peerConnection.createOffer().then(sdp => {
+ peerConnection.setLocalDescription(sdp).then(() => { setTimeout(() => {
+ if (!iceGatheringDone) {
+ iceGatheringDone = true
+ connectToAvatarService(peerConnection)
+ }
+ }, 2000) })
+ })
+}
+
+// Connect to TTS Avatar Service
+function connectToAvatarService(peerConnection) {
+ let localSdp = btoa(JSON.stringify(peerConnection.localDescription))
+ let headers = {
+ 'ClientId': clientId,
+ 'AvatarCharacter': document.getElementById('talkingAvatarCharacter').value,
+ 'AvatarStyle': document.getElementById('talkingAvatarStyle').value,
+ 'IsCustomAvatar': document.getElementById('customizedAvatar').checked
+ }
+
+ if (document.getElementById('azureOpenAIDeploymentName').value !== '') {
+ headers['AoaiDeploymentName'] = document.getElementById('azureOpenAIDeploymentName').value
+ }
+
+ if (document.getElementById('enableOyd').checked && document.getElementById('azureCogSearchIndexName').value !== '') {
+ headers['CognitiveSearchIndexName'] = document.getElementById('azureCogSearchIndexName').value
+ }
+
+ if (document.getElementById('ttsVoice').value !== '') {
+ headers['TtsVoice'] = document.getElementById('ttsVoice').value
+ }
+
+ if (document.getElementById('customVoiceEndpointId').value !== '') {
+ headers['CustomVoiceEndpointId'] = document.getElementById('customVoiceEndpointId').value
+ }
+
+ if (document.getElementById('personalVoiceSpeakerProfileID').value !== '') {
+ headers['PersonalVoiceSpeakerProfileId'] = document.getElementById('personalVoiceSpeakerProfileID').value
+ }
+
+ fetch('/api/connectAvatar', {
+ method: 'POST',
+ headers: headers,
+ body: localSdp
+ })
+ .then(response => {
+ if (response.ok) {
+ response.text().then(text => {
+ const remoteSdp = text
+ peerConnection.setRemoteDescription(new RTCSessionDescription(JSON.parse(atob(remoteSdp))))
+ })
+ } else {
+ document.getElementById('startSession').disabled = false;
+ document.getElementById('configuration').hidden = false;
+ throw new Error(`Failed connecting to the Avatar service: ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+// Handle user query. Send user query to the chat API and display the response.
+function handleUserQuery(userQuery) {
+ fetch('/api/chat', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId,
+ 'SystemPrompt': document.getElementById('prompt').value,
+ 'Content-Type': 'text/plain'
+ },
+ body: userQuery
+ })
+ .then(response => {
+ if (!response.ok) {
+ throw new Error(`Chat API response status: ${response.status} ${response.statusText}`)
+ }
+
+ let chatHistoryTextArea = document.getElementById('chatHistory')
+ chatHistoryTextArea.innerHTML += 'Assistant: '
+
+ const reader = response.body.getReader()
+
+ // Function to recursively read chunks from the stream
+ function read() {
+ return reader.read().then(({ value, done }) => {
+ // Check if there is still data to read
+ if (done) {
+ // Stream complete
+ return
+ }
+
+ // Process the chunk of data (value)
+ let chunkString = new TextDecoder().decode(value, { stream: true })
+
+ chatHistoryTextArea.innerHTML += `${chunkString}`
+ chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
+
+ // Continue reading the next chunk
+ return read()
+ })
+ }
+
+ // Start reading the stream
+ return read()
+ })
+}
+
+// Handle local video. If the user is not speaking for 15 seconds, switch to local video.
+function handleLocalVideo() {
+ if (lastSpeakTime === undefined) {
+ return
+ }
+
+ let currentTime = new Date()
+ if (currentTime - lastSpeakTime > 15000) {
+ if (document.getElementById('useLocalVideoForIdle').checked && sessionActive && !isSpeaking) {
+ disconnectAvatar()
+ document.getElementById('localVideo').hidden = false
+ document.getElementById('remoteVideo').style.width = '0.1px'
+ sessionActive = false
+ }
+ }
+}
+
+// Check whether the avatar video stream is hung
+function checkHung() {
+ // Check whether the avatar video stream is hung, by checking whether the video time is advancing
+ let videoElement = document.getElementById('videoPlayer')
+ if (videoElement !== null && videoElement !== undefined && sessionActive) {
+ let videoTime = videoElement.currentTime
+ setTimeout(() => {
+ // Check whether the video time is advancing
+ if (videoElement.currentTime === videoTime) {
+ // Check whether the session is active to avoid duplicatedly triggering reconnect
+ if (sessionActive) {
+ sessionActive = false
+ if (document.getElementById('autoReconnectAvatar').checked) {
+ console.log(`[${(new Date()).toISOString()}] The video stream got disconnected, need reconnect.`)
+ connectAvatar()
+ createSpeechRecognizer()
+ }
+ }
+ }
+ }, 2000)
+ }
+}
+
+window.onload = () => {
+ clientId = document.getElementById('clientId').value
+ setInterval(() => {
+ checkHung()
+ }, 2000) // Check session activity every 2 seconds
+}
+
+window.startSession = () => {
+ createSpeechRecognizer()
+ if (document.getElementById('useLocalVideoForIdle').checked) {
+ document.getElementById('startSession').disabled = true
+ document.getElementById('configuration').hidden = true
+ document.getElementById('microphone').disabled = false
+ document.getElementById('stopSession').disabled = false
+ document.getElementById('localVideo').hidden = false
+ document.getElementById('remoteVideo').style.width = '0.1px'
+ document.getElementById('chatHistory').hidden = false
+ document.getElementById('showTypeMessage').disabled = false
+ return
+ }
+
+ connectAvatar()
+}
+
+window.stopSpeaking = () => {
+ document.getElementById('stopSpeaking').disabled = true
+
+ fetch('/api/stopSpeaking', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId
+ },
+ body: ''
+ })
+ .then(response => {
+ if (response.ok) {
+ console.log('Successfully stopped speaking.')
+ } else {
+ throw new Error(`Failed to stop speaking: ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+window.stopSession = () => {
+ document.getElementById('startSession').disabled = false
+ document.getElementById('microphone').disabled = true
+ document.getElementById('stopSession').disabled = true
+ document.getElementById('configuration').hidden = false
+ document.getElementById('chatHistory').hidden = true
+ document.getElementById('showTypeMessage').checked = false
+ document.getElementById('showTypeMessage').disabled = true
+ document.getElementById('userMessageBox').hidden = true
+ if (document.getElementById('useLocalVideoForIdle').checked) {
+ document.getElementById('localVideo').hidden = true
+ }
+
+ disconnectAvatar(true)
+}
+
+window.clearChatHistory = () => {
+ fetch('/api/chat/clearHistory', {
+ method: 'POST',
+ headers: {
+ 'ClientId': clientId,
+ 'SystemPrompt': document.getElementById('prompt').value
+ },
+ body: ''
+ })
+ .then(response => {
+ if (response.ok) {
+ document.getElementById('chatHistory').innerHTML = ''
+ } else {
+ throw new Error(`Failed to clear chat history: ${response.status} ${response.statusText}`)
+ }
+ })
+}
+
+window.microphone = () => {
+ if (document.getElementById('microphone').innerHTML === 'Stop Microphone') {
+ // Stop microphone
+ document.getElementById('microphone').disabled = true
+ speechRecognizer.stopContinuousRecognitionAsync(
+ () => {
+ document.getElementById('microphone').innerHTML = 'Start Microphone'
+ document.getElementById('microphone').disabled = false
+ }, (err) => {
+ console.log("Failed to stop continuous recognition:", err)
+ document.getElementById('microphone').disabled = false
+ })
+
+ return
+ }
+
+ if (document.getElementById('useLocalVideoForIdle').checked) {
+ if (!sessionActive) {
+ connectAvatar()
+ }
+
+ setTimeout(() => {
+ document.getElementById('audioPlayer').play()
+ }, 5000)
+ } else {
+ document.getElementById('audioPlayer').play()
+ }
+
+ document.getElementById('microphone').disabled = true
+ speechRecognizer.recognizing = async (s, e) => {
+ if (isFirstRecognizingEvent && isSpeaking) {
+ window.stopSpeaking()
+ isFirstRecognizingEvent = false
+ }
+ }
+
+ speechRecognizer.recognized = async (s, e) => {
+ console.log("recognized event handler is set.")
+ if (e.result.reason === SpeechSDK.ResultReason.RecognizedSpeech) {
+ let userQuery = e.result.text.trim()
+ if (userQuery === '') {
+ return
+ }
+
+ // Auto stop microphone when a phrase is recognized, when it's not continuous conversation mode
+ if (!document.getElementById('continuousConversation').checked) {
+ document.getElementById('microphone').disabled = true
+ speechRecognizer.stopContinuousRecognitionAsync(
+ () => {
+ document.getElementById('microphone').innerHTML = 'Start Microphone'
+ document.getElementById('microphone').disabled = false
+ }, (err) => {
+ console.log("Failed to stop continuous recognition:", err)
+ document.getElementById('microphone').disabled = false
+ })
+ }
+
+ let chatHistoryTextArea = document.getElementById('chatHistory')
+ if (chatHistoryTextArea.innerHTML !== '' && !chatHistoryTextArea.innerHTML.endsWith('\n\n')) {
+ chatHistoryTextArea.innerHTML += '\n\n'
+ }
+
+ chatHistoryTextArea.innerHTML += "User: " + userQuery + '\n\n'
+ chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
+
+ handleUserQuery(userQuery)
+
+ isFirstRecognizingEvent = true
+ }
+ }
+
+ speechRecognizer.startContinuousRecognitionAsync(
+ () => {
+ console.log('Speech recognition started successfully.');
+ document.getElementById('microphone').innerHTML = 'Stop Microphone'
+ document.getElementById('microphone').disabled = false
+ }, (err) => {
+ console.log("Failed to start continuous recognition:", err)
+ document.getElementById('microphone').disabled = false
+ })
+}
+
+window.updataEnableOyd = () => {
+ if (document.getElementById('enableOyd').checked) {
+ document.getElementById('cogSearchConfig').hidden = false
+ } else {
+ document.getElementById('cogSearchConfig').hidden = true
+ }
+}
+
+window.updateTypeMessageBox = () => {
+ if (document.getElementById('showTypeMessage').checked) {
+ document.getElementById('userMessageBox').hidden = false
+ document.getElementById('userMessageBox').addEventListener('keyup', (e) => {
+ if (e.key === 'Enter') {
+ const userQuery = document.getElementById('userMessageBox').value
+ if (userQuery !== '') {
+ let chatHistoryTextArea = document.getElementById('chatHistory')
+ if (chatHistoryTextArea.innerHTML !== '' && !chatHistoryTextArea.innerHTML.endsWith('\n\n')) {
+ chatHistoryTextArea.innerHTML += '\n\n'
+ }
+
+ chatHistoryTextArea.innerHTML += "User: " + userQuery.trim('\n') + '\n\n'
+ chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
+
+ handleUserQuery(userQuery.trim('\n'))
+ document.getElementById('userMessageBox').value = ''
+ }
+ }
+ })
+ } else {
+ document.getElementById('userMessageBox').hidden = true
+ }
+}
+
+window.updateLocalVideoForIdle = () => {
+ if (document.getElementById('useLocalVideoForIdle').checked) {
+ document.getElementById('showTypeMessageCheckbox').hidden = true
+ } else {
+ document.getElementById('showTypeMessageCheckbox').hidden = false
+ }
+}
From 75d8e479c09ada879b32a9120a4a739249bd8a16 Mon Sep 17 00:00:00 2001
From: yinhew <46698869+yinhew@users.noreply.github.com>
Date: Tue, 17 Sep 2024 09:55:07 +0800
Subject: [PATCH 05/11] [Live Avatar][Python, CSharp] Add logging for latency
(#2587)
---
.../avatar/Controllers/AvatarController.cs | 27 +++++++++
.../csharp/web/avatar/Views/Home/basic.cshtml | 2 +-
.../csharp/web/avatar/Views/Home/chat.cshtml | 12 +++-
samples/csharp/web/avatar/wwwroot/js/chat.js | 60 ++++++++++++++++++-
samples/python/web/avatar/app.py | 19 +++++-
samples/python/web/avatar/chat.html | 10 ++++
samples/python/web/avatar/static/js/chat.js | 58 ++++++++++++++++++
7 files changed, 182 insertions(+), 6 deletions(-)
diff --git a/samples/csharp/web/avatar/Controllers/AvatarController.cs b/samples/csharp/web/avatar/Controllers/AvatarController.cs
index fb16fec40..28e9700db 100644
--- a/samples/csharp/web/avatar/Controllers/AvatarController.cs
+++ b/samples/csharp/web/avatar/Controllers/AvatarController.cs
@@ -484,13 +484,24 @@ public async Task HandleUserQuery(string userQuery, Guid clientId, HttpResponse
#pragma warning restore AOAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
}
+ var aoaiStartTime = DateTime.Now;
var chatUpdates = chatClient.CompleteChatStreaming(messages, chatOptions);
+ var isFirstChunk = true;
+ var isFirstSentence = true;
foreach (var chatUpdate in chatUpdates)
{
foreach (var contentPart in chatUpdate.ContentUpdate)
{
var responseToken = contentPart.Text;
+ if (isFirstChunk)
+ {
+ var aoaiFirstTokenLatency = (int)(DateTime.Now.Subtract(aoaiStartTime).TotalMilliseconds + 0.5);
+ Console.WriteLine($"AOAI first token latency: {aoaiFirstTokenLatency}ms");
+ await httpResponse.WriteAsync($"{aoaiFirstTokenLatency}");
+ isFirstChunk = false;
+ }
+
if (ClientSettings.OydDocRegex.IsMatch(responseToken))
{
responseToken = ClientSettings.OydDocRegex.Replace(responseToken, string.Empty);
@@ -501,6 +512,14 @@ public async Task HandleUserQuery(string userQuery, Guid clientId, HttpResponse
assistantReply.Append(responseToken);
if (responseToken == "\n" || responseToken == "\n\n")
{
+ if (isFirstSentence)
+ {
+ var aoaiFirstSentenceLatency = (int)(DateTime.Now.Subtract(aoaiStartTime).TotalMilliseconds + 0.5);
+ Console.WriteLine($"AOAI first sentence latency: {aoaiFirstSentenceLatency}ms");
+ await httpResponse.WriteAsync($"{aoaiFirstSentenceLatency}");
+ isFirstSentence = false;
+ }
+
await SpeakWithQueue(spokenSentence.ToString().Trim(), 0, clientId);
spokenSentence.Clear();
}
@@ -514,6 +533,14 @@ public async Task HandleUserQuery(string userQuery, Guid clientId, HttpResponse
{
if (responseToken.StartsWith(punctuation))
{
+ if (isFirstSentence)
+ {
+ var aoaiFirstSentenceLatency = (int)(DateTime.Now.Subtract(aoaiStartTime).TotalMilliseconds + 0.5);
+ Console.WriteLine($"AOAI first sentence latency: {aoaiFirstSentenceLatency}ms");
+ await httpResponse.WriteAsync($"{aoaiFirstSentenceLatency}");
+ isFirstSentence = false;
+ }
+
await SpeakWithQueue(spokenSentence.ToString().Trim(), 0, clientId);
spokenSentence.Clear();
break;
diff --git a/samples/csharp/web/avatar/Views/Home/basic.cshtml b/samples/csharp/web/avatar/Views/Home/basic.cshtml
index 4d8bf3369..16c3d4acf 100644
--- a/samples/csharp/web/avatar/Views/Home/basic.cshtml
+++ b/samples/csharp/web/avatar/Views/Home/basic.cshtml
@@ -46,7 +46,7 @@
Avatar Control Panel
-
+
diff --git a/samples/csharp/web/avatar/Views/Home/chat.cshtml b/samples/csharp/web/avatar/Views/Home/chat.cshtml
index 742f79302..049dfde5e 100644
--- a/samples/csharp/web/avatar/Views/Home/chat.cshtml
+++ b/samples/csharp/web/avatar/Views/Home/chat.cshtml
@@ -80,8 +80,18 @@
background-color: transparent;
overflow: hidden;" hidden>
+
+
+
-
+
diff --git a/samples/csharp/web/avatar/wwwroot/js/chat.js b/samples/csharp/web/avatar/wwwroot/js/chat.js
index 7d35eacf2..c87af236a 100644
--- a/samples/csharp/web/avatar/wwwroot/js/chat.js
+++ b/samples/csharp/web/avatar/wwwroot/js/chat.js
@@ -7,8 +7,12 @@ var speechRecognizer
var peerConnection
var isSpeaking = false
var sessionActive = false
+var recognitionStartedTime
+var chatResponseReceivedTime
var lastSpeakTime
var isFirstRecognizingEvent = true
+var firstTokenLatencyRegex = new RegExp(/(\d+)<\/FTL>/)
+var firstSentenceLatencyRegex = new RegExp(/(\d+)<\/FSL>/)
// Connect to avatar service
function connectAvatar() {
@@ -134,6 +138,7 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
document.getElementById('stopSession').disabled = false
document.getElementById('remoteVideo').style.width = '960px'
document.getElementById('chatHistory').hidden = false
+ document.getElementById('latencyLog').hidden = false
document.getElementById('showTypeMessage').disabled = false
if (document.getElementById('useLocalVideoForIdle').checked) {
@@ -155,6 +160,16 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
console.log("[" + (new Date()).toISOString() + "] WebRTC event received: " + e.data)
if (e.data.includes("EVENT_TYPE_SWITCH_TO_SPEAKING")) {
+ if (chatResponseReceivedTime !== undefined) {
+ let speakStartTime = new Date()
+ let ttsLatency = speakStartTime - chatResponseReceivedTime
+ console.log(`TTS latency: ${ttsLatency} ms`)
+ let latencyLogTextArea = document.getElementById('latencyLog')
+ latencyLogTextArea.innerHTML += `TTS latency: ${ttsLatency} ms\n\n`
+ latencyLogTextArea.scrollTop = latencyLogTextArea.scrollHeight
+ chatResponseReceivedTime = undefined
+ }
+
isSpeaking = true
document.getElementById('stopSpeaking').disabled = false
} else if (e.data.includes("EVENT_TYPE_SWITCH_TO_IDLE")) {
@@ -254,6 +269,7 @@ function connectToAvatarService(peerConnection) {
// Handle user query. Send user query to the chat API and display the response.
function handleUserQuery(userQuery) {
+ let chatRequestSentTime = new Date()
fetch('/api/chat', {
method: 'POST',
headers: {
@@ -285,6 +301,32 @@ function handleUserQuery(userQuery) {
// Process the chunk of data (value)
let chunkString = new TextDecoder().decode(value, { stream: true })
+ if (firstTokenLatencyRegex.test(chunkString)) {
+ let aoaiFirstTokenLatency = parseInt(firstTokenLatencyRegex.exec(chunkString)[0].replace('', '').replace('', ''))
+ // console.log(`AOAI first token latency: ${aoaiFirstTokenLatency} ms`)
+ chunkString = chunkString.replace(firstTokenLatencyRegex, '')
+ if (chunkString === '') {
+ return read()
+ }
+ }
+
+ if (firstSentenceLatencyRegex.test(chunkString)) {
+ let aoaiFirstSentenceLatency = parseInt(firstSentenceLatencyRegex.exec(chunkString)[0].replace('', '').replace('', ''))
+ chatResponseReceivedTime = new Date()
+ let chatLatency = chatResponseReceivedTime - chatRequestSentTime
+ let appServiceLatency = chatLatency - aoaiFirstSentenceLatency
+ console.log(`App service latency: ${appServiceLatency} ms`)
+ console.log(`AOAI latency: ${aoaiFirstSentenceLatency} ms`)
+ let latencyLogTextArea = document.getElementById('latencyLog')
+ latencyLogTextArea.innerHTML += `App service latency: ${appServiceLatency} ms\n`
+ latencyLogTextArea.innerHTML += `AOAI latency: ${aoaiFirstSentenceLatency} ms\n`
+ latencyLogTextArea.scrollTop = latencyLogTextArea.scrollHeight
+ chunkString = chunkString.replace(firstSentenceLatencyRegex, '')
+ if (chunkString === '') {
+ return read()
+ }
+ }
+
chatHistoryTextArea.innerHTML += `${chunkString}`
chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
@@ -355,6 +397,7 @@ window.startSession = () => {
document.getElementById('localVideo').hidden = false
document.getElementById('remoteVideo').style.width = '0.1px'
document.getElementById('chatHistory').hidden = false
+ document.getElementById('latencyLog').hidden = false
document.getElementById('showTypeMessage').disabled = false
return
}
@@ -387,6 +430,7 @@ window.stopSession = () => {
document.getElementById('stopSession').disabled = true
document.getElementById('configuration').hidden = false
document.getElementById('chatHistory').hidden = true
+ document.getElementById('latencyLog').hidden = true
document.getElementById('showTypeMessage').checked = false
document.getElementById('showTypeMessage').disabled = true
document.getElementById('userMessageBox').hidden = true
@@ -409,6 +453,7 @@ window.clearChatHistory = () => {
.then(response => {
if (response.ok) {
document.getElementById('chatHistory').innerHTML = ''
+ document.getElementById('latencyLog').innerHTML = ''
} else {
throw new Error(`Failed to clear chat history: ${response.status} ${response.statusText}`)
}
@@ -452,13 +497,20 @@ window.microphone = () => {
}
speechRecognizer.recognized = async (s, e) => {
- console.log("recognized event handler is set.")
if (e.result.reason === SpeechSDK.ResultReason.RecognizedSpeech) {
let userQuery = e.result.text.trim()
if (userQuery === '') {
return
}
+ let recognitionResultReceivedTime = new Date()
+ let speechFinishedOffset = (e.result.offset + e.result.duration) / 10000
+ let sttLatency = recognitionResultReceivedTime - recognitionStartedTime - speechFinishedOffset
+ console.log(`STT latency: ${sttLatency} ms`)
+ let latencyLogTextArea = document.getElementById('latencyLog')
+ latencyLogTextArea.innerHTML += `STT latency: ${sttLatency} ms\n`
+ latencyLogTextArea.scrollTop = latencyLogTextArea.scrollHeight
+
// Auto stop microphone when a phrase is recognized, when it's not continuous conversation mode
if (!document.getElementById('continuousConversation').checked) {
document.getElementById('microphone').disabled = true
@@ -486,9 +538,9 @@ window.microphone = () => {
}
}
+ recognitionStartedTime = new Date()
speechRecognizer.startContinuousRecognitionAsync(
() => {
- console.log('Speech recognition started successfully.');
document.getElementById('microphone').innerHTML = 'Stop Microphone'
document.getElementById('microphone').disabled = false
}, (err) => {
@@ -520,6 +572,10 @@ window.updateTypeMessageBox = () => {
chatHistoryTextArea.innerHTML += "User: " + userQuery.trim('\n') + '\n\n'
chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
+ if (isSpeaking) {
+ window.stopSpeaking()
+ }
+
handleUserQuery(userQuery.trim('\n'))
document.getElementById('userMessageBox').value = ''
}
diff --git a/samples/python/web/avatar/app.py b/samples/python/web/avatar/app.py
index 032f6b137..000714244 100644
--- a/samples/python/web/avatar/app.py
+++ b/samples/python/web/avatar/app.py
@@ -381,19 +381,29 @@ def handleUserQuery(user_query: str, client_id: uuid.UUID):
messages=messages,
extra_body={ 'data_sources' : data_sources } if len(data_sources) > 0 else None,
stream=True)
- aoai_reponse_time = datetime.datetime.now(pytz.UTC)
- print(f"AOAI latency: {(aoai_reponse_time - aoai_start_time).total_seconds() * 1000}ms")
+ is_first_chunk = True
+ is_first_sentence = True
for chunk in response:
if len(chunk.choices) > 0:
response_token = chunk.choices[0].delta.content
if response_token is not None:
# Log response_token here if need debug
+ if is_first_chunk:
+ first_token_latency_ms = round((datetime.datetime.now(pytz.UTC) - aoai_start_time).total_seconds() * 1000)
+ print(f"AOAI first token latency: {first_token_latency_ms}ms")
+ yield f"{first_token_latency_ms}"
+ is_first_chunk = False
if oyd_doc_regex.search(response_token):
response_token = oyd_doc_regex.sub('', response_token).strip()
yield response_token # yield response token to client as display text
assistant_reply += response_token # build up the assistant message
if response_token == '\n' or response_token == '\n\n':
+ if is_first_sentence:
+ first_sentence_latency_ms = round((datetime.datetime.now(pytz.UTC) - aoai_start_time).total_seconds() * 1000)
+ print(f"AOAI first sentence latency: {first_sentence_latency_ms}ms")
+ yield f"{first_sentence_latency_ms}"
+ is_first_sentence = False
speakWithQueue(spoken_sentence.strip(), 0, client_id)
spoken_sentence = ''
else:
@@ -402,6 +412,11 @@ def handleUserQuery(user_query: str, client_id: uuid.UUID):
if len(response_token) == 1 or len(response_token) == 2:
for punctuation in sentence_level_punctuations:
if response_token.startswith(punctuation):
+ if is_first_sentence:
+ first_sentence_latency_ms = round((datetime.datetime.now(pytz.UTC) - aoai_start_time).total_seconds() * 1000)
+ print(f"AOAI first sentence latency: {first_sentence_latency_ms}ms")
+ yield f"{first_sentence_latency_ms}"
+ is_first_sentence = False
speakWithQueue(spoken_sentence.strip(), 0, client_id)
spoken_sentence = ''
break
diff --git a/samples/python/web/avatar/chat.html b/samples/python/web/avatar/chat.html
index 59b9ad190..ff378dcb1 100644
--- a/samples/python/web/avatar/chat.html
+++ b/samples/python/web/avatar/chat.html
@@ -78,6 +78,16 @@ Avatar Configuration
background-color: transparent;
overflow: hidden;" hidden>
+
+
+
diff --git a/samples/python/web/avatar/static/js/chat.js b/samples/python/web/avatar/static/js/chat.js
index ea380c5b4..c87af236a 100644
--- a/samples/python/web/avatar/static/js/chat.js
+++ b/samples/python/web/avatar/static/js/chat.js
@@ -7,8 +7,12 @@ var speechRecognizer
var peerConnection
var isSpeaking = false
var sessionActive = false
+var recognitionStartedTime
+var chatResponseReceivedTime
var lastSpeakTime
var isFirstRecognizingEvent = true
+var firstTokenLatencyRegex = new RegExp(/(\d+)<\/FTL>/)
+var firstSentenceLatencyRegex = new RegExp(/(\d+)<\/FSL>/)
// Connect to avatar service
function connectAvatar() {
@@ -134,6 +138,7 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
document.getElementById('stopSession').disabled = false
document.getElementById('remoteVideo').style.width = '960px'
document.getElementById('chatHistory').hidden = false
+ document.getElementById('latencyLog').hidden = false
document.getElementById('showTypeMessage').disabled = false
if (document.getElementById('useLocalVideoForIdle').checked) {
@@ -155,6 +160,16 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
console.log("[" + (new Date()).toISOString() + "] WebRTC event received: " + e.data)
if (e.data.includes("EVENT_TYPE_SWITCH_TO_SPEAKING")) {
+ if (chatResponseReceivedTime !== undefined) {
+ let speakStartTime = new Date()
+ let ttsLatency = speakStartTime - chatResponseReceivedTime
+ console.log(`TTS latency: ${ttsLatency} ms`)
+ let latencyLogTextArea = document.getElementById('latencyLog')
+ latencyLogTextArea.innerHTML += `TTS latency: ${ttsLatency} ms\n\n`
+ latencyLogTextArea.scrollTop = latencyLogTextArea.scrollHeight
+ chatResponseReceivedTime = undefined
+ }
+
isSpeaking = true
document.getElementById('stopSpeaking').disabled = false
} else if (e.data.includes("EVENT_TYPE_SWITCH_TO_IDLE")) {
@@ -254,6 +269,7 @@ function connectToAvatarService(peerConnection) {
// Handle user query. Send user query to the chat API and display the response.
function handleUserQuery(userQuery) {
+ let chatRequestSentTime = new Date()
fetch('/api/chat', {
method: 'POST',
headers: {
@@ -285,6 +301,32 @@ function handleUserQuery(userQuery) {
// Process the chunk of data (value)
let chunkString = new TextDecoder().decode(value, { stream: true })
+ if (firstTokenLatencyRegex.test(chunkString)) {
+ let aoaiFirstTokenLatency = parseInt(firstTokenLatencyRegex.exec(chunkString)[0].replace('', '').replace('', ''))
+ // console.log(`AOAI first token latency: ${aoaiFirstTokenLatency} ms`)
+ chunkString = chunkString.replace(firstTokenLatencyRegex, '')
+ if (chunkString === '') {
+ return read()
+ }
+ }
+
+ if (firstSentenceLatencyRegex.test(chunkString)) {
+ let aoaiFirstSentenceLatency = parseInt(firstSentenceLatencyRegex.exec(chunkString)[0].replace('', '').replace('', ''))
+ chatResponseReceivedTime = new Date()
+ let chatLatency = chatResponseReceivedTime - chatRequestSentTime
+ let appServiceLatency = chatLatency - aoaiFirstSentenceLatency
+ console.log(`App service latency: ${appServiceLatency} ms`)
+ console.log(`AOAI latency: ${aoaiFirstSentenceLatency} ms`)
+ let latencyLogTextArea = document.getElementById('latencyLog')
+ latencyLogTextArea.innerHTML += `App service latency: ${appServiceLatency} ms\n`
+ latencyLogTextArea.innerHTML += `AOAI latency: ${aoaiFirstSentenceLatency} ms\n`
+ latencyLogTextArea.scrollTop = latencyLogTextArea.scrollHeight
+ chunkString = chunkString.replace(firstSentenceLatencyRegex, '')
+ if (chunkString === '') {
+ return read()
+ }
+ }
+
chatHistoryTextArea.innerHTML += `${chunkString}`
chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
@@ -355,6 +397,7 @@ window.startSession = () => {
document.getElementById('localVideo').hidden = false
document.getElementById('remoteVideo').style.width = '0.1px'
document.getElementById('chatHistory').hidden = false
+ document.getElementById('latencyLog').hidden = false
document.getElementById('showTypeMessage').disabled = false
return
}
@@ -387,6 +430,7 @@ window.stopSession = () => {
document.getElementById('stopSession').disabled = true
document.getElementById('configuration').hidden = false
document.getElementById('chatHistory').hidden = true
+ document.getElementById('latencyLog').hidden = true
document.getElementById('showTypeMessage').checked = false
document.getElementById('showTypeMessage').disabled = true
document.getElementById('userMessageBox').hidden = true
@@ -409,6 +453,7 @@ window.clearChatHistory = () => {
.then(response => {
if (response.ok) {
document.getElementById('chatHistory').innerHTML = ''
+ document.getElementById('latencyLog').innerHTML = ''
} else {
throw new Error(`Failed to clear chat history: ${response.status} ${response.statusText}`)
}
@@ -458,6 +503,14 @@ window.microphone = () => {
return
}
+ let recognitionResultReceivedTime = new Date()
+ let speechFinishedOffset = (e.result.offset + e.result.duration) / 10000
+ let sttLatency = recognitionResultReceivedTime - recognitionStartedTime - speechFinishedOffset
+ console.log(`STT latency: ${sttLatency} ms`)
+ let latencyLogTextArea = document.getElementById('latencyLog')
+ latencyLogTextArea.innerHTML += `STT latency: ${sttLatency} ms\n`
+ latencyLogTextArea.scrollTop = latencyLogTextArea.scrollHeight
+
// Auto stop microphone when a phrase is recognized, when it's not continuous conversation mode
if (!document.getElementById('continuousConversation').checked) {
document.getElementById('microphone').disabled = true
@@ -485,6 +538,7 @@ window.microphone = () => {
}
}
+ recognitionStartedTime = new Date()
speechRecognizer.startContinuousRecognitionAsync(
() => {
document.getElementById('microphone').innerHTML = 'Stop Microphone'
@@ -518,6 +572,10 @@ window.updateTypeMessageBox = () => {
chatHistoryTextArea.innerHTML += "User: " + userQuery.trim('\n') + '\n\n'
chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight
+ if (isSpeaking) {
+ window.stopSpeaking()
+ }
+
handleUserQuery(userQuery.trim('\n'))
document.getElementById('userMessageBox').value = ''
}
From 203bd467e6849825f3823c435f9a91691bb34f0f Mon Sep 17 00:00:00 2001
From: Henry van der Vegte
Date: Wed, 18 Sep 2024 16:13:39 +0200
Subject: [PATCH 06/11] [IngestionClient] Use dependency injection for
BatchClient, add User-Agent request header (#2592)
---
.../ingestion-client/Connector/BatchClient.cs | 53 +++++++++++--------
.../FetchTranscription/Config/AppConfig.cs | 2 +
.../FetchTranscription/FetchTranscription.cs | 11 +++-
.../FetchTranscription/Program.cs | 11 ++++
.../TranscriptionProcessor.cs | 22 ++++----
.../Config/AppConfig.cs | 2 +
.../StartTranscriptionByTimer/Program.cs | 11 ++++
.../StartTranscriptionHelper.cs | 6 ++-
.../ingestion-client/infra/main.bicep | 4 +-
.../ingestion-client/infra/main.json | 10 ++--
10 files changed, 93 insertions(+), 39 deletions(-)
diff --git a/samples/ingestion/ingestion-client/Connector/BatchClient.cs b/samples/ingestion/ingestion-client/Connector/BatchClient.cs
index c5791a3e6..4fc471942 100644
--- a/samples/ingestion/ingestion-client/Connector/BatchClient.cs
+++ b/samples/ingestion/ingestion-client/Connector/BatchClient.cs
@@ -17,7 +17,7 @@ namespace Connector
using Polly;
using Polly.Retry;
- public static class BatchClient
+ public class BatchClient
{
private const string TranscriptionsBasePath = "speechtotext/v3.0/Transcriptions/";
@@ -29,36 +29,42 @@ public static class BatchClient
private static readonly TimeSpan GetFilesTimeout = TimeSpan.FromMinutes(5);
- private static readonly HttpClient HttpClient = new HttpClient() { Timeout = Timeout.InfiniteTimeSpan };
-
private static readonly AsyncRetryPolicy RetryPolicy =
Policy
.Handle(e => e is HttpStatusCodeException || e is HttpRequestException)
.WaitAndRetryAsync(MaxNumberOfRetries, retryAttempt => TimeSpan.FromSeconds(5));
- public static Task GetTranscriptionReportFileFromSasAsync(string sasUri)
+ private readonly HttpClient httpClient;
+
+ public BatchClient(IHttpClientFactory httpClientFactory)
+ {
+ ArgumentNullException.ThrowIfNull(httpClientFactory, nameof(httpClientFactory));
+ this.httpClient = httpClientFactory.CreateClient(nameof(BatchClient));
+ }
+
+ public Task GetTranscriptionReportFileFromSasAsync(string sasUri)
{
- return GetAsync(sasUri, null, DefaultTimeout);
+ return this.GetAsync(sasUri, null, DefaultTimeout);
}
- public static Task GetSpeechTranscriptFromSasAsync(string sasUri)
+ public Task GetSpeechTranscriptFromSasAsync(string sasUri)
{
- return GetAsync(sasUri, null, DefaultTimeout);
+ return this.GetAsync(sasUri, null, DefaultTimeout);
}
- public static Task GetTranscriptionAsync(string transcriptionLocation, string subscriptionKey)
+ public Task GetTranscriptionAsync(string transcriptionLocation, string subscriptionKey)
{
- return GetAsync(transcriptionLocation, subscriptionKey, DefaultTimeout);
+ return this.GetAsync