-
Notifications
You must be signed in to change notification settings - Fork 343
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(voicea-highlight): added-highlights #3807
base: next
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -804,6 +804,8 @@ const generalControlsDtmfTones = document.querySelector('#gc-dtmf-tones'); | |
const generalControlsDtmfStatus = document.querySelector('#gc-dtmf-status'); | ||
const generalToggleTranscription = document.querySelector('#gc-toggle-transcription'); | ||
const generalTranscriptionContent = document.querySelector('#gc-transcription-content'); | ||
const generalHighlightTranscription = document.querySelector('#gc-toggle-highlights'); | ||
const generalHighlightContent = document.querySelector('#gc-highlights-content'); | ||
|
||
const sourceDevicesGetMedia = document.querySelector('#sd-get-media-devices'); | ||
const sourceDevicesAudioInput = document.querySelector('#sd-audio-input-devices'); | ||
|
@@ -1104,6 +1106,44 @@ async function toggleTranscription(enable = false){ | |
} | ||
} | ||
|
||
async function toggleHighlights() { | ||
const isEnabled = generalHighlightTranscription.getAttribute('data-enabled') === "true"; | ||
if (isEnabled) { | ||
try { | ||
await meeting.toggleHighlighting(!isEnabled); | ||
generalHighlightTranscription.setAttribute('data-enabled', "false"); | ||
generalHighlightTranscription.innerText = "Start Highlight"; | ||
generalHighlightContent.innerHTML = 'Highlight Content: Webex Assistant must be enabled, check the console!'; | ||
} | ||
catch (e) { | ||
console.error("Error stopping highlight", e); | ||
} | ||
} | ||
else { | ||
let firsttime = generalHighlightTranscription.dataset.firsttime; | ||
if (firsttime === undefined) { | ||
const currentMeeting = getCurrentMeeting(); | ||
if (currentMeeting) { | ||
generalHighlightContent.innerHTML = ''; | ||
meeting.on('meeting:highlight-created', (payload) => { | ||
generalHighlightContent.innerHTML = `\n${JSON.stringify(payload,null,4)}`; | ||
}); | ||
} | ||
generalHighlightTranscription.dataset.firsttime = "yes"; | ||
} | ||
try { | ||
generalHighlightContent.innerHTML = ''; | ||
await meeting.toggleHighlighting(!isEnabled); | ||
generalHighlightTranscription.setAttribute('data-enabled', "true"); | ||
generalHighlightTranscription.innerText = "Stop Highlight"; | ||
} | ||
catch (e) { | ||
generalHighlightContent.innerHTML = 'Highlight Content: Webex Assistant must be enabled, check the console!'; | ||
console.error("Error starting highlight", e); | ||
} | ||
} | ||
} | ||
|
||
function setTranscriptEvents() { | ||
const meeting = getCurrentMeeting(); | ||
|
||
|
@@ -1144,6 +1184,9 @@ function setTranscriptEvents() { | |
meeting.on('meeting:receiveTranscription:stopped', () => { | ||
generalToggleTranscription.innerText = "Start Transcription"; | ||
generalTranscriptionContent.innerHTML = 'Transcription Content: Webex Assistant must be enabled, check the console!'; | ||
generalHighlightTranscription.setAttribute('data-enabled', "false"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. IMPORTANT - Here we need to disable the transcription button and replace the text as when transcription is stopped highlighting no longer sends events. I'm not a 100% sure about this method (open to any suggestions). However, since this works for the sample app - I feel it should work. |
||
generalHighlightTranscription.innerText = "Start Highlight"; | ||
generalHighlightContent.innerHTML = 'Highlight Content: Webex Assistant must be enabled, check the console!'; | ||
}); | ||
} | ||
else { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -851,13 +851,17 @@ <h2 class="collapsible"> | |
<div class="btn-group u-mb"> | ||
<button id="gc-toggle-transcription" type="button" | ||
onclick="toggleTranscription()" class="btn-code" data-enabled="false">Start Transcription</button> | ||
<button id="gc-toggle-highlights" type="button" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Creating new UI for this change. |
||
onclick="toggleHighlights()" class="btn-code" data-enabled="false">Show Highlights</button> | ||
<br> | ||
Caption Language: <select id="voicea-caption-language" style="display: inline;" disabled></select><button id="voicea-caption-language-btn" onclick="setCaptionLanguage()" disabled>meeting.setCaptionLanguage()</button><br/> | ||
Spoken Language: <select id="voicea-spoken-language" style="display: inline;" disabled></select><button id="voicea-spoken-language-btn" onclick="setSpokenLanguage()" disabled>meeting.setSpokenLanguage()</button> <p id="only-host-spoken-language" class="hidden">Only Meeting Host can Set Spoken Language</p> | ||
<br> | ||
</div> | ||
<textarea id="gc-transcription-content" | ||
disabled="">Transcription Content: Webex Assistant must be enabled, check the console!</textarea> | ||
<textarea id="gc-highlights-content" | ||
disabled="">Highlighting: Webex Assistant must be enabled, check the console!</textarea> | ||
</div> | ||
</fieldset> | ||
</div> | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,11 @@ | ||
import * as WebexCore from '@webex/webex-core'; | ||
|
||
import VoiceaChannel from './voicea'; | ||
import type {MeetingTranscriptPayload} from './voicea.types'; | ||
import type {MeetingTranscriptPayload, MeetingHighlightPayload} from './voicea.types'; | ||
|
||
WebexCore.registerInternalPlugin('voicea', VoiceaChannel, {}); | ||
|
||
export {default} from './voicea'; | ||
export {type MeetingTranscriptPayload}; | ||
export {type MeetingHighlightPayload}; | ||
export {EVENT_TRIGGERS, TURN_ON_CAPTION_STATUS} from './constants'; |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -365,6 +365,7 @@ export const EVENT_TRIGGERS = { | |
MEETING_STOPPED_RECEIVING_TRANSCRIPTION: 'meeting:receiveTranscription:stopped', | ||
MEETING_MANUAL_CAPTION_UPDATED: 'meeting:manualCaptionControl:updated', | ||
MEETING_CAPTION_RECEIVED: 'meeting:caption-received', | ||
MEETING_HIGHLIGHT_CREATED: 'meeting:highlight-created', | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. New event to be emitted from meeting object to the user. |
||
}; | ||
|
||
export const EVENT_TYPES = { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -42,9 +42,10 @@ import { | |
EVENT_TRIGGERS as VOICEAEVENTS, | ||
TURN_ON_CAPTION_STATUS, | ||
type MeetingTranscriptPayload, | ||
type MeetingHighlightPayload, | ||
} from '@webex/internal-plugin-voicea'; | ||
|
||
import {processNewCaptions} from './voicea-meeting'; | ||
import {processHighlightCreated, processNewCaptions} from './voicea-meeting'; | ||
|
||
import { | ||
MeetingNotActiveError, | ||
|
@@ -183,6 +184,17 @@ export type CaptionData = { | |
speaker: string; | ||
}; | ||
|
||
export type Highlight = { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We would need a new type for highlights object. |
||
id: string; | ||
meta: { | ||
label: string; | ||
source: string; | ||
}; | ||
text: string; | ||
timestamp: string; | ||
speaker: any; | ||
}; | ||
|
||
export type Transcription = { | ||
languageOptions: { | ||
captionLanguages?: string; // list of supported caption languages from backend | ||
|
@@ -201,6 +213,7 @@ export type Transcription = { | |
isCaptioning: boolean; | ||
speakerProxy: Map<string, any>; | ||
interimCaptions: Map<string, CaptionData>; | ||
highlights: Array<Highlight>; | ||
}; | ||
|
||
export type LocalStreams = { | ||
|
@@ -680,6 +693,21 @@ export default class Meeting extends StatelessWebexPlugin { | |
interimCaptions: this.transcription.interimCaptions, | ||
}); | ||
}, | ||
[VOICEAEVENTS.HIGHLIGHT_CREATED]: (data: MeetingHighlightPayload) => { | ||
processHighlightCreated({data, meeting: this}); | ||
|
||
LoggerProxy.logger.debug( | ||
`${EventsUtil.getScopeLog({ | ||
file: 'meeting/index', | ||
function: 'setUpVoiceaListeners', | ||
})}event#${EVENT_TRIGGERS.MEETING_HIGHLIGHT_CREATED}` | ||
); | ||
|
||
// @ts-ignore | ||
this.trigger(EVENT_TRIGGERS.MEETING_HIGHLIGHT_CREATED, { | ||
highlights: this.transcription.highlights, | ||
}); | ||
}, | ||
}; | ||
|
||
private addMediaData: { | ||
|
@@ -4944,6 +4972,48 @@ export default class Meeting extends StatelessWebexPlugin { | |
} | ||
} | ||
|
||
/** | ||
* This method will toggle the highlights for the current meeting if the meeting has enabled/supports Webex Assistant | ||
* @param {Boolean} activate flag to enable/disable highlights | ||
* @param {Object} options object with spokenlanguage setting | ||
* @public | ||
* @returns {Promise<void>} a promise to open the WebSocket connection | ||
*/ | ||
public async toggleHighlighting(activate: boolean, options?: {spokenLanguage?: string}) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rather than a separate |
||
if (this.isJoined() && this.isTranscriptionSupported()) { | ||
LoggerProxy.logger.info( | ||
'Meeting:index#toggleHighlighting --> Attempting to enable highlights!' | ||
); | ||
|
||
try { | ||
if (activate) { | ||
// @ts-ignore | ||
this.webex.internal.voicea.on( | ||
VOICEAEVENTS.HIGHLIGHT_CREATED, | ||
this.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED] | ||
); | ||
} else { | ||
// @ts-ignore | ||
this.webex.internal.voicea.off( | ||
VOICEAEVENTS.HIGHLIGHT_CREATED, | ||
this.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED] | ||
); | ||
} | ||
// @ts-ignore | ||
await this.webex.internal.voicea.toggleTranscribing(activate, options?.spokenLanguage); | ||
} catch (error) { | ||
LoggerProxy.logger.error(`Meeting:index#toggleHighlighting --> ${error}`); | ||
Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.RECEIVE_TRANSCRIPTION_FAILURE, { | ||
correlation_id: this.correlationId, | ||
reason: error.message, | ||
stack: error.stack, | ||
}); | ||
} | ||
} else { | ||
throw new Error('This operation is not allowed in your org. Contact org administrator.'); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We throw this error if we find that transcriptions are not enabled. Usually happens if the old webex assistant is not enabled for the org. |
||
} | ||
} | ||
|
||
/** | ||
* Callback called when a relay event is received from meeting LLM Connection | ||
* @param {RelayEvent} e Event object coming from LLM Connection | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,7 @@ | ||
import {type MeetingTranscriptPayload} from '@webex/internal-plugin-voicea'; | ||
import { | ||
type MeetingTranscriptPayload, | ||
type MeetingHighlightPayload, | ||
} from '@webex/internal-plugin-voicea'; | ||
|
||
export const getSpeaker = (members, csis = []) => | ||
Object.values(members).find((member: any) => { | ||
|
@@ -118,3 +121,41 @@ export const processNewCaptions = ({ | |
} | ||
transcriptData.interimCaptions[transcriptId] = interimTranscriptionIds; | ||
}; | ||
|
||
export const processHighlightCreated = ({ | ||
data, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We simply process the highlights voicea gives back and pack it into the meetings object as an array. |
||
meeting, | ||
}: { | ||
data: MeetingHighlightPayload; | ||
meeting: any; | ||
}) => { | ||
const transcriptData = meeting.transcription; | ||
|
||
if (!transcriptData.highlights) { | ||
transcriptData.highlights = []; | ||
} | ||
|
||
const csisKey = data.csis && data.csis.length > 0 ? data.csis[0] : undefined; | ||
const {needsCaching, speaker} = getSpeakerFromProxyOrStore({ | ||
meetingMembers: meeting.members.membersCollection.members, | ||
transcriptData, | ||
csisKey, | ||
}); | ||
|
||
if (needsCaching) { | ||
transcriptData.speakerProxy[csisKey] = speaker; | ||
} | ||
|
||
const highlightCreated = { | ||
id: data.highlightId, | ||
meta: { | ||
label: data.highlightLabel, | ||
source: data.highlightSource, | ||
}, | ||
text: data.text, | ||
timestamp: data.timestamp, | ||
speaker, | ||
}; | ||
|
||
meeting.transcription.highlights.push(highlightCreated); | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1188,6 +1188,49 @@ describe('plugin-meetings', () => { | |
}); | ||
}); | ||
|
||
describe('#toggleHighlighting', () => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added tests to check all the flows (toggling). Had to mock a lot of data. |
||
beforeEach(() => { | ||
webex.internal.voicea.on = sinon.stub(); | ||
webex.internal.voicea.off = sinon.stub(); | ||
webex.internal.voicea.listenToEvents = sinon.stub(); | ||
webex.internal.voicea.toggleTranscribing = sinon.stub(); | ||
meeting.isTranscriptionSupported = sinon.stub(); | ||
}); | ||
|
||
afterEach(() => { | ||
sinon.restore(); | ||
}); | ||
|
||
it('should reject if transcription is not supported', (done) => { | ||
meeting.isTranscriptionSupported.returns(false); | ||
|
||
meeting.toggleHighlighting(true).catch((error) => { | ||
assert.equal(error.message, 'This operation is not allowed in your org. Contact org administrator.'); | ||
done(); | ||
}); | ||
}); | ||
|
||
it('should enable highlights when meeting is joined and transcription is supported', async () => { | ||
meeting.joinedWith = { | ||
state: 'JOINED', | ||
}; | ||
meeting.isTranscriptionSupported.returns(true); | ||
await meeting.toggleHighlighting(true); | ||
assert.calledOnce(meeting.webex.internal.voicea.toggleTranscribing); | ||
assert.equal(webex.internal.voicea.on.callCount, 1); | ||
}); | ||
|
||
it('should disable highlights when meeting is joined and transcription is supported', async () => { | ||
meeting.joinedWith = { | ||
state: 'JOINED', | ||
}; | ||
meeting.isTranscriptionSupported.returns(true); | ||
await meeting.toggleHighlighting(false) | ||
assert.calledOnce(meeting.webex.internal.voicea.toggleTranscribing); | ||
assert.equal(webex.internal.voicea.off.callCount, 1); | ||
}); | ||
}); | ||
|
||
describe('#setCaptionLanguage', () => { | ||
beforeEach(() => { | ||
meeting.isTranscriptionSupported = sinon.stub(); | ||
|
@@ -1360,6 +1403,14 @@ describe('plugin-meetings', () => { | |
EVENT_TRIGGERS.MEETING_CAPTION_RECEIVED | ||
); | ||
}); | ||
|
||
it('should trigger meeting:highlight-created event', () => { | ||
meeting.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED]({}); | ||
assert.calledWith( | ||
meeting.trigger, | ||
EVENT_TRIGGERS.MEETING_HIGHLIGHT_CREATED | ||
); | ||
}); | ||
}); | ||
|
||
describe('#isReactionsSupported', () => { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using toggle is an easier way for users to access highlighting.
Here, we check for the button data status and if it is set as 'true', it indicated that it was turned on and hence needs to be switched off - hence the first block is that flow while the second block is the flow for turning on the highlights.