ref(transcriptions): refactor transcriptions api

Extend IFrame API to allow adding a transcriber in the room without the subtitles needing to be visible.
Allow transcription chunk messages to be passed through the IFrame API if a transcriber is present.
Clean-up transcription messages sent through the IFrame API to not include timeout field and possible conflicting states (stable / unstable /final)

modules/API/API.js

@@ -466,8 +466,8 @@ function initCommands() {
         'toggle-subtitles': () => {
             APP.store.dispatch(toggleRequestingSubtitles());
         },
-        'set-subtitles': enabled => {
-            APP.store.dispatch(setRequestingSubtitles(enabled));
+        'set-subtitles': (enabled, displaySubtitles, language) => {
+            APP.store.dispatch(setRequestingSubtitles(enabled, displaySubtitles, language));
         },
         'toggle-tile-view': () => {
             sendAnalytics(createApiEvent('tile-view.toggled'));

react/features/base/i18n/i18next.ts

@@ -40,14 +40,6 @@ export const LANGUAGES: Array<string> = Object.keys(LANGUAGES_RESOURCES);
  */
 export const TRANSLATION_LANGUAGES: Array<string> = Object.keys(TRANSLATION_LANGUAGES_RESOURCES);
 
-/**
- * The available/supported translation languages head. (Languages displayed on the top ).
- *
- * @public
- * @type {Array<string>}
- */
-export const TRANSLATION_LANGUAGES_HEAD: Array<string> = [ 'en' ];
-
 /**
  * The default language.
  *
@@ -58,6 +50,14 @@ export const TRANSLATION_LANGUAGES_HEAD: Array<string> = [ 'en' ];
  */
 export const DEFAULT_LANGUAGE = 'en';
 
+/**
+ * The available/supported translation languages head. (Languages displayed on the top ).
+ *
+ * @public
+ * @type {Array<string>}
+ */
+export const TRANSLATION_LANGUAGES_HEAD: Array<string> = [ DEFAULT_LANGUAGE ];
+
 /**
  * The options to initialize i18next with.
  *

react/features/mobile/external-api/middleware.ts

@@ -379,9 +379,10 @@ function _registerForNativeEvents(store: IStore) {
         dispatch(sendMessage(message));
     });
 
-    eventEmitter.addListener(ExternalAPI.SET_CLOSED_CAPTIONS_ENABLED, ({ enabled }: any) => {
-        dispatch(setRequestingSubtitles(enabled));
-    });
+    eventEmitter.addListener(ExternalAPI.SET_CLOSED_CAPTIONS_ENABLED,
+        ({ enabled, displaySubtitles, language }: any) => {
+            dispatch(setRequestingSubtitles(enabled, displaySubtitles, language));
+        });
 
     eventEmitter.addListener(ExternalAPI.TOGGLE_CAMERA, () => {
         dispatch(toggleCameraFacingMode());

react/features/subtitles/actionTypes.ts

@@ -34,18 +34,6 @@ export const REMOVE_TRANSCRIPT_MESSAGE = 'REMOVE_TRANSCRIPT_MESSAGE';
  */
 export const UPDATE_TRANSCRIPT_MESSAGE = 'UPDATE_TRANSCRIPT_MESSAGE';
 
-/**
- * The type of (redux) action which indicates that a transcript with an
- * given message_id to be added or updated is received.
- *
- * {
- *      type: UPDATE_TRANSLATION_LANGUAGE,
- *      transcriptMessageID: string,
- *      newTranscriptMessage: Object
- * }
- */
-export const UPDATE_TRANSLATION_LANGUAGE = 'UPDATE_TRANSLATION_LANGUAGE';
-
 /**
  * The type of (redux) action which indicates that the user pressed the
  * ClosedCaption button, to either enable or disable subtitles based on the

react/features/subtitles/actions.any.ts

@@ -1,10 +1,11 @@
+import { DEFAULT_LANGUAGE } from '../base/i18n/i18next';
+
 import {
     ENDPOINT_MESSAGE_RECEIVED,
     REMOVE_TRANSCRIPT_MESSAGE,
     SET_REQUESTING_SUBTITLES,
     TOGGLE_REQUESTING_SUBTITLES,
-    UPDATE_TRANSCRIPT_MESSAGE,
-    UPDATE_TRANSLATION_LANGUAGE
+    UPDATE_TRANSCRIPT_MESSAGE
 } from './actionTypes';
 
 /**
@@ -80,29 +81,23 @@ export function toggleRequestingSubtitles() {
  * Signals that the local user has enabled or disabled the subtitles.
  *
  * @param {boolean} enabled - The new state of the subtitles.
+ * @param {boolean} displaySubtitles - Whether to display subtitles or not.
+ * @param {string} language - The language of the subtitles.
  * @returns {{
  *    type: SET_REQUESTING_SUBTITLES,
- *    enabled: boolean
+ *    enabled: boolean,
+ *    displaySubtitles: boolean,
+ *    language: string
  * }}
  */
-export function setRequestingSubtitles(enabled: boolean) {
+export function setRequestingSubtitles(
+        enabled: boolean,
+        displaySubtitles = true,
+        language: string | null = DEFAULT_LANGUAGE) {
     return {
         type: SET_REQUESTING_SUBTITLES,
-        enabled
-    };
-}
-
-/**
- * Signals that the local user has selected language for the translation.
- *
- * @param {string | null} value - The selected language for translation.
- * @returns {{
- *      type: UPDATE_TRANSLATION_LANGUAGE
- * }}
- */
-export function updateTranslationLanguage(value: string | null) {
-    return {
-        type: UPDATE_TRANSLATION_LANGUAGE,
-        value
+        displaySubtitles,
+        enabled,
+        language: `translation-languages:${language}`
     };
 }

react/features/subtitles/actions.web.ts

@@ -8,9 +8,7 @@ export * from './actions.any';
 /**
  * Signals that the local user has toggled the LanguageSelector button.
  *
- * @returns {{
- *      type: UPDATE_TRANSLATION_LANGUAGE
- * }}
+ * @returns {Function}
  */
 export function toggleLanguageSelectorDialog() {
     return function(dispatch: IStore['dispatch']) {

react/features/subtitles/components/AbstractCaptions.tsx

@@ -9,7 +9,12 @@ import { IReduxState } from '../../app/types';
 export interface IAbstractCaptionsProps {
 
     /**
-     * Whether local participant is requesting to see subtitles.
+     * Whether local participant is displaying subtitles.
+     */
+    _displaySubtitles: boolean;
+
+    /**
+     * Whether local participant is requesting subtitles.
      */
     _requestingSubtitles: boolean;
 
@@ -34,9 +39,9 @@ export class AbstractCaptions<P extends IAbstractCaptionsProps> extends Componen
      * @returns {ReactElement}
      */
     render(): any {
-        const { _requestingSubtitles, _transcripts } = this.props;
+        const { _displaySubtitles, _requestingSubtitles, _transcripts } = this.props;
 
-        if (!_requestingSubtitles || !_transcripts || !_transcripts.size) {
+        if (!_requestingSubtitles || !_displaySubtitles || !_transcripts || !_transcripts.size) {
             return null;
         }
 
@@ -125,10 +130,11 @@ function _constructTranscripts(state: IReduxState): Map<string, string> {
  * }}
  */
 export function _abstractMapStateToProps(state: IReduxState) {
-    const { _requestingSubtitles } = state['features/subtitles'];
+    const { _displaySubtitles, _requestingSubtitles } = state['features/subtitles'];
     const transcripts = _constructTranscripts(state);
 
     return {
+        _displaySubtitles,
         _requestingSubtitles,
 
         // avoid re-renders by setting to prop new empty Map instances.

react/features/subtitles/components/AbstractLanguageSelectorDialog.tsx

@@ -1,4 +1,4 @@
-import React, { ComponentType, useCallback, useState } from 'react';
+import React, { ComponentType, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 
@@ -7,7 +7,7 @@ import {
     TRANSLATION_LANGUAGES,
     TRANSLATION_LANGUAGES_HEAD
 } from '../../base/i18n/i18next';
-import { setRequestingSubtitles, updateTranslationLanguage } from '../actions.any';
+import { setRequestingSubtitles } from '../actions.any';
 
 
 export interface IAbstractLanguageSelectorDialogProps {
@@ -33,7 +33,7 @@ const AbstractLanguageSelectorDialog = (Component: ComponentType<IAbstractLangua
     const noLanguageLabel = 'transcribing.subtitlesOff';
 
     const language = useSelector((state: IReduxState) => state['features/subtitles']._language);
-    const [ subtitles, setSubtitles ] = useState(language || noLanguageLabel);
+    const subtitles = language ?? noLanguageLabel;
 
     const transcription = useSelector((state: IReduxState) => state['features/base/config'].transcription);
     const translationLanguagesHead = transcription?.translationLanguagesHead ?? TRANSLATION_LANGUAGES_HEAD;
@@ -60,10 +60,10 @@ const AbstractLanguageSelectorDialog = (Component: ComponentType<IAbstractLangua
 
     const onLanguageSelected = useCallback((value: string) => {
         const selectedLanguage = value === noLanguageLabel ? null : value;
+        const enabled = Boolean(selectedLanguage);
+        const displaySubtitles = enabled;
 
-        setSubtitles(value);
-        dispatch(updateTranslationLanguage(selectedLanguage));
-        dispatch(setRequestingSubtitles(Boolean(selectedLanguage)));
+        dispatch(setRequestingSubtitles(enabled, displaySubtitles, selectedLanguage));
     }, [ language ]);
 
     return (

react/features/subtitles/middleware.ts

@@ -55,12 +55,15 @@ MiddlewareRegistry.register(store => next => action => {
     case ENDPOINT_MESSAGE_RECEIVED:
         return _endpointMessageReceived(store, next, action);
 
-    case TOGGLE_REQUESTING_SUBTITLES:
-        _requestingSubtitlesChange(store);
+    case TOGGLE_REQUESTING_SUBTITLES: {
+        const state = store.getState()['features/subtitles'];
+        const toggledValue = !state._requestingSubtitles;
+
+        _requestingSubtitlesChange(store, toggledValue, state._language);
         break;
+    }
     case SET_REQUESTING_SUBTITLES:
-        _requestingSubtitlesChange(store);
-        _requestingSubtitlesSet(store, action.enabled);
+        _requestingSubtitlesChange(store, action.enabled, action.language);
         break;
     }
 
@@ -125,10 +128,10 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
             // We update the previous transcript message with the same
             // message ID or adds a new transcript message if it does not
             // exist in the map.
+            const existingMessage = state['features/subtitles']._transcriptMessages.get(transcriptMessageID);
             const newTranscriptMessage: any = {
-                ...state['features/subtitles']._transcriptMessages
-                        .get(transcriptMessageID)
-                    || { participantName }
+                participantName,
+                clearTimeOut: existingMessage?.clearTimeOut
             };
 
             _setClearerOnTranscriptMessage(dispatch,
@@ -144,7 +147,6 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
                 // stable field of the state and remove the previously
                 // unstable results
                 newTranscriptMessage.stable = text;
-                newTranscriptMessage.unstable = undefined;
 
             } else {
                 // Otherwise, this result has an unstable result, which we
@@ -157,9 +159,13 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
 
             // Notify the external API too.
             if (typeof APP !== 'undefined') {
+                const sanitizedTranscriptMessage = { ...newTranscriptMessage };
+
+                delete sanitizedTranscriptMessage.clearTimeOut;
+
                 APP.API.notifyTranscriptionChunkReceived({
                     messageID: transcriptMessageID,
-                    ...newTranscriptMessage
+                    ...sanitizedTranscriptMessage
                 });
             }
         }
@@ -175,45 +181,29 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
  * and Jigasi to decide whether the transcriber needs to be in the room.
  *
  * @param {Store} store - The redux store.
+ * @param {boolean} enabled - Whether subtitles should be enabled or not.
+ * @param {string} language - The language to use for translation.
  * @private
  * @returns {void}
  */
-function _requestingSubtitlesChange({ getState }: IStore) {
+function _requestingSubtitlesChange(
+        { getState }: IStore,
+        enabled: boolean,
+        language?: string | null) {
     const state = getState();
-    const { _language } = state['features/subtitles'];
     const { conference } = state['features/base/conference'];
 
-    const requestingSubtitles = Boolean(_language);
-
     conference?.setLocalParticipantProperty(
         P_NAME_REQUESTING_TRANSCRIPTION,
-        requestingSubtitles);
+        enabled);
 
-    if (requestingSubtitles) {
+    if (enabled && language) {
         conference?.setLocalParticipantProperty(
             P_NAME_TRANSLATION_LANGUAGE,
-            _language.replace('translation-languages:', ''));
+            language.replace('translation-languages:', ''));
     }
 }
 
-/**
- * Set the local property 'requestingTranscription'. This will cause Jicofo
- * and Jigasi to decide whether the transcriber needs to be in the room.
- *
- * @param {Store} store - The redux store.
- * @param {boolean} enabled - The new state of the subtitles.
- * @private
- * @returns {void}
- */
-function _requestingSubtitlesSet({ getState }: IStore, enabled: boolean) {
-    const state = getState();
-    const { conference } = state['features/base/conference'];
-
-    conference?.setLocalParticipantProperty(
-        P_NAME_REQUESTING_TRANSCRIPTION,
-        enabled);
-}
-
 /**
  * Set a timeout on a TranscriptMessage object so it clears itself when it's not
  * updated.

react/features/subtitles/reducer.ts

@@ -2,13 +2,14 @@ import ReducerRegistry from '../base/redux/ReducerRegistry';
 
 import {
     REMOVE_TRANSCRIPT_MESSAGE,
-    SET_REQUESTING_SUBTITLES, UPDATE_TRANSCRIPT_MESSAGE, UPDATE_TRANSLATION_LANGUAGE
+    SET_REQUESTING_SUBTITLES, TOGGLE_REQUESTING_SUBTITLES, UPDATE_TRANSCRIPT_MESSAGE
 } from './actionTypes';
 
 /**
  * Default State for 'features/transcription' feature.
  */
 const defaultState = {
+    _displaySubtitles: true,
     _transcriptMessages: new Map(),
     _requestingSubtitles: false,
     _language: null
@@ -22,6 +23,7 @@ interface ITranscriptMessage {
 }
 
 export interface ISubtitlesState {
+    _displaySubtitles: boolean;
     _language: string | null;
     _requestingSubtitles: boolean;
     _transcriptMessages: Map<string, ITranscriptMessage> | any;
@@ -38,15 +40,17 @@ ReducerRegistry.register<ISubtitlesState>('features/subtitles', (
         return _removeTranscriptMessage(state, action);
     case UPDATE_TRANSCRIPT_MESSAGE:
         return _updateTranscriptMessage(state, action);
-    case UPDATE_TRANSLATION_LANGUAGE:
+    case SET_REQUESTING_SUBTITLES:
         return {
             ...state,
-            _language: action.value
+            _displaySubtitles: action.displaySubtitles,
+            _language: action.language,
+            _requestingSubtitles: action.enabled
         };
-    case SET_REQUESTING_SUBTITLES:
+    case TOGGLE_REQUESTING_SUBTITLES:
         return {
             ...state,
-            _requestingSubtitles: action.enabled
+            _requestingSubtitles: !state._requestingSubtitles
         };
     }