From 453540dbf9089dde12875b2dfa5181d0e56cbda4 Mon Sep 17 00:00:00 2001 From: adhmenon Date: Wed, 4 Sep 2024 11:58:32 +0530 Subject: [PATCH 1/2] feat(voice-highlight): added-highlights --- .../@webex/plugin-meetings/src/constants.ts | 1 + .../plugin-meetings/src/meeting/index.ts | 72 ++++++++++++++++++- .../src/meeting/voicea-meeting.ts | 43 ++++++++++- .../test/unit/spec/meeting/index.js | 51 +++++++++++++ .../test/unit/spec/meeting/voicea-meeting.ts | 47 ++++++++++++ 5 files changed, 212 insertions(+), 2 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/constants.ts b/packages/@webex/plugin-meetings/src/constants.ts index 26fbfcd6c7b..fcd24f74f98 100644 --- a/packages/@webex/plugin-meetings/src/constants.ts +++ b/packages/@webex/plugin-meetings/src/constants.ts @@ -365,6 +365,7 @@ export const EVENT_TRIGGERS = { MEETING_STOPPED_RECEIVING_TRANSCRIPTION: 'meeting:receiveTranscription:stopped', MEETING_MANUAL_CAPTION_UPDATED: 'meeting:manualCaptionControl:updated', MEETING_CAPTION_RECEIVED: 'meeting:caption-received', + MEETING_HIGHLIGHT_CREATED: 'meeting:highlight-created', }; export const EVENT_TYPES = { diff --git a/packages/@webex/plugin-meetings/src/meeting/index.ts b/packages/@webex/plugin-meetings/src/meeting/index.ts index e255e3f633b..18655baa6c8 100644 --- a/packages/@webex/plugin-meetings/src/meeting/index.ts +++ b/packages/@webex/plugin-meetings/src/meeting/index.ts @@ -42,9 +42,10 @@ import { EVENT_TRIGGERS as VOICEAEVENTS, TURN_ON_CAPTION_STATUS, type MeetingTranscriptPayload, + type MeetingHighlightPayload, } from '@webex/internal-plugin-voicea'; -import {processNewCaptions} from './voicea-meeting'; +import {processHighlightCreated, processNewCaptions} from './voicea-meeting'; import { MeetingNotActiveError, @@ -183,6 +184,17 @@ export type CaptionData = { speaker: string; }; +export type Highlight = { + id: string; + meta: { + label: string; + source: string; + }; + text: string; + timestamp: string; + speaker: any; +}; + export type Transcription = { languageOptions: { captionLanguages?: string; // list of supported caption languages from backend @@ -201,6 +213,7 @@ export type Transcription = { isCaptioning: boolean; speakerProxy: Map; interimCaptions: Map; + highlights: Array; }; export type LocalStreams = { @@ -680,6 +693,21 @@ export default class Meeting extends StatelessWebexPlugin { interimCaptions: this.transcription.interimCaptions, }); }, + [VOICEAEVENTS.HIGHLIGHT_CREATED]: (data: MeetingHighlightPayload) => { + processHighlightCreated({data, meeting: this}); + + LoggerProxy.logger.debug( + `${EventsUtil.getScopeLog({ + file: 'meeting/index', + function: 'setUpVoiceaListeners', + })}event#${EVENT_TRIGGERS.MEETING_HIGHLIGHT_CREATED}` + ); + + // @ts-ignore + this.trigger(EVENT_TRIGGERS.MEETING_HIGHLIGHT_CREATED, { + highlights: this.transcription.highlights, + }); + }, }; private addMediaData: { @@ -4944,6 +4972,48 @@ export default class Meeting extends StatelessWebexPlugin { } } + /** + * This method will toggle the highlights for the current meeting if the meeting has enabled/supports Webex Assistant + * @param {Boolean} activate flag to enable/disable highlights + * @param {Object} options object with spokenlanguage setting + * @public + * @returns {Promise} a promise to open the WebSocket connection + */ + public async toggleHighlighting(activate: boolean, options?: {spokenLanguage?: string}) { + if (this.isJoined() && this.isTranscriptionSupported()) { + LoggerProxy.logger.info( + 'Meeting:index#toggleHighlighting --> Attempting to enable highlights!' + ); + + try { + if (activate) { + // @ts-ignore + this.webex.internal.voicea.on( + VOICEAEVENTS.HIGHLIGHT_CREATED, + this.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED] + ); + } else { + // @ts-ignore + this.webex.internal.voicea.off( + VOICEAEVENTS.HIGHLIGHT_CREATED, + this.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED] + ); + } + // @ts-ignore + await this.webex.internal.voicea.toggleTranscribing(activate, options?.spokenLanguage); + } catch (error) { + LoggerProxy.logger.error(`Meeting:index#toggleHighlighting --> ${error}`); + Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.RECEIVE_TRANSCRIPTION_FAILURE, { + correlation_id: this.correlationId, + reason: error.message, + stack: error.stack, + }); + } + } else { + throw new Error('This operation is not allowed in your org. Contact org administrator.'); + } + } + /** * Callback called when a relay event is received from meeting LLM Connection * @param {RelayEvent} e Event object coming from LLM Connection diff --git a/packages/@webex/plugin-meetings/src/meeting/voicea-meeting.ts b/packages/@webex/plugin-meetings/src/meeting/voicea-meeting.ts index ae5e0e2b273..3e067a71a3b 100644 --- a/packages/@webex/plugin-meetings/src/meeting/voicea-meeting.ts +++ b/packages/@webex/plugin-meetings/src/meeting/voicea-meeting.ts @@ -1,4 +1,7 @@ -import {type MeetingTranscriptPayload} from '@webex/internal-plugin-voicea'; +import { + type MeetingTranscriptPayload, + type MeetingHighlightPayload, +} from '@webex/internal-plugin-voicea'; export const getSpeaker = (members, csis = []) => Object.values(members).find((member: any) => { @@ -118,3 +121,41 @@ export const processNewCaptions = ({ } transcriptData.interimCaptions[transcriptId] = interimTranscriptionIds; }; + +export const processHighlightCreated = ({ + data, + meeting, +}: { + data: MeetingHighlightPayload; + meeting: any; +}) => { + const transcriptData = meeting.transcription; + + if (!transcriptData.highlights) { + transcriptData.highlights = []; + } + + const csisKey = data.csis && data.csis.length > 0 ? data.csis[0] : undefined; + const {needsCaching, speaker} = getSpeakerFromProxyOrStore({ + meetingMembers: meeting.members.membersCollection.members, + transcriptData, + csisKey, + }); + + if (needsCaching) { + transcriptData.speakerProxy[csisKey] = speaker; + } + + const highlightCreated = { + id: data.highlightId, + meta: { + label: data.highlightLabel, + source: data.highlightSource, + }, + text: data.text, + timestamp: data.timestamp, + speaker, + }; + + meeting.transcription.highlights.push(highlightCreated); +}; diff --git a/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js b/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js index 24e352c815f..29324ddab67 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js @@ -1188,6 +1188,49 @@ describe('plugin-meetings', () => { }); }); + describe('#toggleHighlighting', () => { + beforeEach(() => { + webex.internal.voicea.on = sinon.stub(); + webex.internal.voicea.off = sinon.stub(); + webex.internal.voicea.listenToEvents = sinon.stub(); + webex.internal.voicea.toggleTranscribing = sinon.stub(); + meeting.isTranscriptionSupported = sinon.stub(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should reject if transcription is not supported', (done) => { + meeting.isTranscriptionSupported.returns(false); + + meeting.toggleHighlighting(true).catch((error) => { + assert.equal(error.message, 'This operation is not allowed in your org. Contact org administrator.'); + done(); + }); + }); + + it('should enable highlights when meeting is joined and transcription is supported', async () => { + meeting.joinedWith = { + state: 'JOINED', + }; + meeting.isTranscriptionSupported.returns(true); + await meeting.toggleHighlighting(true); + assert.calledOnce(meeting.webex.internal.voicea.toggleTranscribing); + assert.equal(webex.internal.voicea.on.callCount, 1); + }); + + it('should disable highlights when meeting is joined and transcription is supported', async () => { + meeting.joinedWith = { + state: 'JOINED', + }; + meeting.isTranscriptionSupported.returns(true); + await meeting.toggleHighlighting(false) + assert.calledOnce(meeting.webex.internal.voicea.toggleTranscribing); + assert.equal(webex.internal.voicea.off.callCount, 1); + }); + }); + describe('#setCaptionLanguage', () => { beforeEach(() => { meeting.isTranscriptionSupported = sinon.stub(); @@ -1360,6 +1403,14 @@ describe('plugin-meetings', () => { EVENT_TRIGGERS.MEETING_CAPTION_RECEIVED ); }); + + it('should trigger meeting:highlight-created event', () => { + meeting.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED]({}); + assert.calledWith( + meeting.trigger, + EVENT_TRIGGERS.MEETING_HIGHLIGHT_CREATED + ); + }); }); describe('#isReactionsSupported', () => { diff --git a/packages/@webex/plugin-meetings/test/unit/spec/meeting/voicea-meeting.ts b/packages/@webex/plugin-meetings/test/unit/spec/meeting/voicea-meeting.ts index d841cbad207..1c92aea6393 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/meeting/voicea-meeting.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/meeting/voicea-meeting.ts @@ -1,6 +1,7 @@ import { getSpeakerFromProxyOrStore, processNewCaptions, + processHighlightCreated, } from '@webex/plugin-meetings/src/meeting/voicea-meeting'; import {assert} from '@webex/test-helper-chai'; import { expect } from 'chai'; @@ -253,5 +254,51 @@ describe('plugin-meetings', () => { expect(newCaption.speaker).to.deep.equal(speaker); }); }); + + describe('processHighlightCreated', () => { + beforeEach(() => { + fakeVoiceaPayload = { + highlightId: 'highlight1', + highlightLabel: 'Important', + highlightSource: 'User', + text: 'This is a highlight', + timestamp: '2023-10-10T10:00:00Z', + csis: ['3060099329'], + }; + }); + + it('should initialize highlights array if it does not exist', () => { + fakeMeeting.transcription.highlights = undefined; + + processHighlightCreated({data: fakeVoiceaPayload, meeting: fakeMeeting}); + + expect(fakeMeeting.transcription.highlights).to.be.an('array'); + expect(fakeMeeting.transcription.highlights).to.have.lengthOf(1); + }); + + it('should process highlight creation correctly', () => { + processHighlightCreated({data: fakeVoiceaPayload, meeting: fakeMeeting}); + + const csisKey = fakeVoiceaPayload.csis[0]; + const speaker = fakeMeeting.transcription.speakerProxy[csisKey]; + expect(speaker).to.exist; + + const newHighlight = fakeMeeting.transcription.highlights.find( + (highlight) => highlight.id === fakeVoiceaPayload.highlightId + ); + expect(newHighlight).to.exist; + expect(newHighlight).to.deep.include({ + id: fakeVoiceaPayload.highlightId, + meta: { + label: fakeVoiceaPayload.highlightLabel, + source: fakeVoiceaPayload.highlightSource, + }, + text: fakeVoiceaPayload.text, + timestamp: fakeVoiceaPayload.timestamp, + }); + + expect(newHighlight.speaker).to.deep.equal(speaker); + }); + }); }); }); \ No newline at end of file From f0abb1909c96c6a385ae8d153acdfb184b3ebe07 Mon Sep 17 00:00:00 2001 From: adhmenon Date: Wed, 4 Sep 2024 12:13:13 +0530 Subject: [PATCH 2/2] feat(voice-highlight): added-changes-to-sample-app --- docs/samples/browser-plugin-meetings/app.js | 43 +++++++++++++++++++ .../browser-plugin-meetings/index.html | 4 ++ .../internal-plugin-voicea/src/index.ts | 3 +- .../src/voicea.types.ts | 10 +++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/samples/browser-plugin-meetings/app.js b/docs/samples/browser-plugin-meetings/app.js index 2e4db88b0eb..393dd666683 100644 --- a/docs/samples/browser-plugin-meetings/app.js +++ b/docs/samples/browser-plugin-meetings/app.js @@ -804,6 +804,8 @@ const generalControlsDtmfTones = document.querySelector('#gc-dtmf-tones'); const generalControlsDtmfStatus = document.querySelector('#gc-dtmf-status'); const generalToggleTranscription = document.querySelector('#gc-toggle-transcription'); const generalTranscriptionContent = document.querySelector('#gc-transcription-content'); +const generalHighlightTranscription = document.querySelector('#gc-toggle-highlights'); +const generalHighlightContent = document.querySelector('#gc-highlights-content'); const sourceDevicesGetMedia = document.querySelector('#sd-get-media-devices'); const sourceDevicesAudioInput = document.querySelector('#sd-audio-input-devices'); @@ -1104,6 +1106,44 @@ async function toggleTranscription(enable = false){ } } +async function toggleHighlights() { + const isEnabled = generalHighlightTranscription.getAttribute('data-enabled') === "true"; + if (isEnabled) { + try { + await meeting.toggleHighlighting(!isEnabled); + generalHighlightTranscription.setAttribute('data-enabled', "false"); + generalHighlightTranscription.innerText = "Start Highlight"; + generalHighlightContent.innerHTML = 'Highlight Content: Webex Assistant must be enabled, check the console!'; + } + catch (e) { + console.error("Error stopping highlight", e); + } + } + else { + let firsttime = generalHighlightTranscription.dataset.firsttime; + if (firsttime === undefined) { + const currentMeeting = getCurrentMeeting(); + if (currentMeeting) { + generalHighlightContent.innerHTML = ''; + meeting.on('meeting:highlight-created', (payload) => { + generalHighlightContent.innerHTML = `\n${JSON.stringify(payload,null,4)}`; + }); + } + generalHighlightTranscription.dataset.firsttime = "yes"; + } + try { + generalHighlightContent.innerHTML = ''; + await meeting.toggleHighlighting(!isEnabled); + generalHighlightTranscription.setAttribute('data-enabled', "true"); + generalHighlightTranscription.innerText = "Stop Highlight"; + } + catch (e) { + generalHighlightContent.innerHTML = 'Highlight Content: Webex Assistant must be enabled, check the console!'; + console.error("Error starting highlight", e); + } + } +} + function setTranscriptEvents() { const meeting = getCurrentMeeting(); @@ -1144,6 +1184,9 @@ function setTranscriptEvents() { meeting.on('meeting:receiveTranscription:stopped', () => { generalToggleTranscription.innerText = "Start Transcription"; generalTranscriptionContent.innerHTML = 'Transcription Content: Webex Assistant must be enabled, check the console!'; + generalHighlightTranscription.setAttribute('data-enabled', "false"); + generalHighlightTranscription.innerText = "Start Highlight"; + generalHighlightContent.innerHTML = 'Highlight Content: Webex Assistant must be enabled, check the console!'; }); } else { diff --git a/docs/samples/browser-plugin-meetings/index.html b/docs/samples/browser-plugin-meetings/index.html index 1df78750757..9d002bb9d5a 100644 --- a/docs/samples/browser-plugin-meetings/index.html +++ b/docs/samples/browser-plugin-meetings/index.html @@ -851,6 +851,8 @@

+
Caption Language:
Spoken Language: @@ -858,6 +860,8 @@

+ diff --git a/packages/@webex/internal-plugin-voicea/src/index.ts b/packages/@webex/internal-plugin-voicea/src/index.ts index ab6b0539c5f..97e86e7c421 100644 --- a/packages/@webex/internal-plugin-voicea/src/index.ts +++ b/packages/@webex/internal-plugin-voicea/src/index.ts @@ -1,10 +1,11 @@ import * as WebexCore from '@webex/webex-core'; import VoiceaChannel from './voicea'; -import type {MeetingTranscriptPayload} from './voicea.types'; +import type {MeetingTranscriptPayload, MeetingHighlightPayload} from './voicea.types'; WebexCore.registerInternalPlugin('voicea', VoiceaChannel, {}); export {default} from './voicea'; export {type MeetingTranscriptPayload}; +export {type MeetingHighlightPayload}; export {EVENT_TRIGGERS, TURN_ON_CAPTION_STATUS} from './constants'; diff --git a/packages/@webex/internal-plugin-voicea/src/voicea.types.ts b/packages/@webex/internal-plugin-voicea/src/voicea.types.ts index 52ab28f2dcc..41a597fc0d9 100644 --- a/packages/@webex/internal-plugin-voicea/src/voicea.types.ts +++ b/packages/@webex/internal-plugin-voicea/src/voicea.types.ts @@ -103,6 +103,15 @@ type MeetingTranscriptPayload = { transcripts: Array; }; +type MeetingHighlightPayload = { + csis: string; + highlightId: string; + text: string; + highlightLabel: string; + highlightSource: string; + timestamp?: string; +}; + export type { AnnouncementPayload, CaptionLanguageResponse, @@ -111,4 +120,5 @@ export type { Highlight, IVoiceaChannel, MeetingTranscriptPayload, + MeetingHighlightPayload, };