Show subtitles when Jigasi sends transcription results in JSON (#1914)

* Show subtitles when Jigasi sends transcription results in JSON

* fix: Import PropTypes from prop-types.

* apply feedback on initial PR

* Changed Object to Map, alphabetic ordering fixes ,css changes in transcription subtitles

* Sends Map of transcriptMessages as prop to Component

* Documentation fixes and uses config in redux state

* Minor doc fix

* rename feature 'transcription' to 'subtitles'

* Moves subtitles config to interfaceConfig and minor fixes

* minor lint fix

conference.js

@@ -109,6 +109,7 @@ import {
 } from './react/features/overlay';
 import { setSharedVideoStatus } from './react/features/shared-video';
 import { isButtonEnabled } from './react/features/toolbox';
+import { endpointMessageReceived } from './react/features/subtitles';
 
 const logger = require('jitsi-meet-logger').getLogger(__filename);
 
@@ -1875,6 +1876,10 @@ export default {
             }
         );
 
+        room.on(
+            JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
+            (...args) => APP.store.dispatch(endpointMessageReceived(...args)));
+
         room.on(
             JitsiConferenceEvents.LOCK_STATE_CHANGED,
             (...args) => APP.store.dispatch(lockStateChanged(room, ...args)));

config.js

@@ -252,7 +252,6 @@ var config = {
     // maintenance at 01:00 AM GMT,
     // noticeMessage: '',
 
-
     // Stats
     //
 

css/_transcription-subtitles.scss

@@ -0,0 +1,13 @@
+.transcription-subtitles{
+    bottom: 10%;
+    font-size: 16px;
+    font-weight: 1000;
+    opacity: 0.80;
+    position: absolute;
+    text-shadow: 0px 0px 1px rgba(0,0,0,0.3),
+    0px 1px 1px rgba(0,0,0,0.3),
+    1px 0px 1px rgba(0,0,0,0.3),
+    0px 0px 1px rgba(0,0,0,0.3);
+    width: 100%;
+    z-index: $zindex2;
+}

css/main.scss

@@ -77,5 +77,5 @@
 @import 'unsupported-browser/main';
 @import 'modals/invite/add-people';
 @import 'deep-linking/main';
-
+@import 'transcription-subtitles';
 /* Modules END */

interface_config.js

@@ -80,6 +80,14 @@ var interfaceConfig = {
     DISABLE_FOCUS_INDICATOR: false,
     DISABLE_DOMINANT_SPEAKER_INDICATOR: false,
 
+    /**
+     * Whether the speech to text transcription subtitles panel is disabled.
+     * If {@code undefined}, defaults to {@code false}.
+     *
+     * @type {boolean}
+     */
+    DISABLE_TRANSCRIPTION_SUBTITLES: false,
+
     /**
      * Whether the ringing sound in the call/ring overlay is disabled. If
      * {@code undefined}, defaults to {@code false}.

react/features/base/conference/actions.js

@@ -54,7 +54,6 @@ import {
     getCurrentConference,
     sendLocalParticipant
 } from './functions';
-
 import type { Dispatch } from 'redux';
 
 const logger = require('jitsi-meet-logger').getLogger(__filename);

react/features/large-video/components/LargeVideo.web.js

@@ -4,6 +4,7 @@ import PropTypes from 'prop-types';
 import React, { Component } from 'react';
 
 import { Watermarks } from '../../base/react';
+import { TranscriptionSubtitles } from '../../subtitles/';
 
 import Labels from './Labels';
 
@@ -70,6 +71,8 @@ export default class LargeVideo extends Component<*> {
                             muted = { true } />
                     </div>
                 </div>
+                { interfaceConfig.DISABLE_TRANSCRIPTION_SUBTITLES
+                    ? null : <TranscriptionSubtitles /> }
                 <span id = 'localConnectionMessage' />
                 { this.props.hideVideoQualityLabel
                     ? null : <Labels /> }

react/features/subtitles/actionTypes.js

@@ -0,0 +1,46 @@
+/**
+ * The type of (redux) action which indicates that a transcript with
+ * a new message_id is received.
+ *
+ * {
+ *      type: ADD_TRANSCRIPT_MESSAGE,
+ *      transcriptMessageID: string,
+ *      participantName: string
+ * }
+ */
+export const ADD_TRANSCRIPT_MESSAGE = Symbol('ADD_TRANSCRIPT_MESSAGE');
+
+/**
+ * The type of (redux) action which indicates that an endpoint message
+ * sent by another participant to the data channel is received.
+ *
+ * {
+ *     type: ENDPOINT_MESSAGE_RECEIVED,
+ *     participant: Object,
+ *     json: Object
+ * }
+ */
+export const ENDPOINT_MESSAGE_RECEIVED = Symbol('ENDPOINT_MESSAGE_RECEIVED');
+
+/**
+ * The type of (redux) action which indicates that an existing transcript
+ * has to be removed from the state.
+ *
+ * {
+ *      type: REMOVE_TRANSCRIPT_MESSAGE,
+ *      transciptMessageID: string,
+ * }
+ */
+export const REMOVE_TRANSCRIPT_MESSAGE = Symbol('REMOVE_TRANSCRIPT_MESSAGE');
+
+/**
+ * The type of (redux) action which indicates that a transcript with an
+ * existing message_id to be updated is received.
+ *
+ * {
+ *      type: UPDATE_TRANSCRIPT_MESSAGE,
+ *      transcriptMessageID: string,
+ *      newTranscriptMessage: Object
+ * }
+ */
+export const UPDATE_TRANSCRIPT_MESSAGE = Symbol('UPDATE_TRANSCRIPT_MESSAGE');

react/features/subtitles/actions.js

@@ -0,0 +1,84 @@
+// @flow
+
+import {
+    ADD_TRANSCRIPT_MESSAGE,
+    ENDPOINT_MESSAGE_RECEIVED,
+    REMOVE_TRANSCRIPT_MESSAGE,
+    UPDATE_TRANSCRIPT_MESSAGE
+} from './actionTypes';
+
+/**
+ * Signals that a transcript with a new message_id is received.
+ *
+ * @param {string} transcriptMessageID - The new message_id.
+ * @param {string} participantName - The participant name of the sender.
+ * @returns {{
+ *      type: ADD_TRANSCRIPT_MESSAGE,
+ *      transcriptMessageID: string,
+ *      participantName: string
+ * }}
+ */
+export function addTranscriptMessage(transcriptMessageID: string,
+        participantName: string) {
+    return {
+        type: ADD_TRANSCRIPT_MESSAGE,
+        transcriptMessageID,
+        participantName
+    };
+}
+
+/**
+ * Signals that a participant sent an endpoint message on the data channel.
+ *
+ * @param {Object} participant - The participant details sending the message.
+ * @param {Object} json - The json carried by the endpoint message.
+ * @returns {{
+ *      type: ENDPOINT_MESSAGE_RECEIVED,
+ *      participant: Object,
+ *      json: Object
+ * }}
+ */
+export function endpointMessageReceived(participant: Object, json: Object) {
+    return {
+        type: ENDPOINT_MESSAGE_RECEIVED,
+        participant,
+        json
+    };
+}
+
+/**
+ * Signals that a transcript has to be removed from the state.
+ *
+ * @param {string} transcriptMessageID - The message_id to be removed.
+ * @returns {{
+ *      type: REMOVE_TRANSCRIPT_MESSAGE,
+ *      transcriptMessageID: string,
+ * }}
+ */
+export function removeTranscriptMessage(transcriptMessageID: string) {
+    return {
+        type: REMOVE_TRANSCRIPT_MESSAGE,
+        transcriptMessageID
+    };
+}
+
+/**
+ * Signals that a transcript with an existing message_id to be updated
+ * is received.
+ *
+ * @param {string} transcriptMessageID -The transcript message_id to be updated.
+ * @param {Object} newTranscriptMessage - The updated transcript message.
+ * @returns {{
+ *      type: UPDATE_TRANSCRIPT_MESSAGE,
+ *      transcriptMessageID: string,
+ *      newTranscriptMessage: Object
+ * }}
+ */
+export function updateTranscriptMessage(transcriptMessageID: string,
+        newTranscriptMessage: Object) {
+    return {
+        type: UPDATE_TRANSCRIPT_MESSAGE,
+        transcriptMessageID,
+        newTranscriptMessage
+    };
+}

react/features/subtitles/components/TranscriptionSubtitles.web.js

@@ -0,0 +1,78 @@
+// @flow
+
+import React, { Component } from 'react';
+import { connect } from 'react-redux';
+
+/**
+ * The type of the React {@code Component} props of
+ * {@link TranscriptionSubtitles}.
+ */
+type Props = {
+
+    /**
+     * Map of transcriptMessageID's with corresponding transcriptMessage.
+     */
+    _transcriptMessages: Map<string, Object>
+};
+
+/**
+ * React {@code Component} which can display speech-to-text results from
+ * Jigasi as subtitles.
+ */
+class TranscriptionSubtitles extends Component<Props> {
+
+    /**
+     * Implements React's {@link Component#render()}.
+     *
+     * @inheritdoc
+     * @returns {ReactElement}
+     */
+    render() {
+        const paragraphs = [];
+
+        for (const [ transcriptMessageID, transcriptMessage ]
+            of this.props._transcriptMessages) {
+            let text;
+
+            if (transcriptMessage) {
+                text = `${transcriptMessage.participantName}: `;
+
+                if (transcriptMessage.final) {
+                    text += transcriptMessage.final;
+                } else {
+                    const stable = transcriptMessage.stable || '';
+                    const unstable = transcriptMessage.unstable || '';
+
+                    text += stable + unstable;
+                }
+                paragraphs.push(
+                    <p key = { transcriptMessageID }> { text } </p>
+                );
+            }
+        }
+
+        return (
+            <div className = 'transcription-subtitles' >
+                { paragraphs }
+            </div>
+        );
+    }
+}
+
+
+/**
+ * Maps the transcriptionSubtitles in the Redux state to the associated
+ * props of {@code TranscriptionSubtitles}.
+ *
+ * @param {Object} state - The Redux state.
+ * @private
+ * @returns {{
+ *     _transcriptMessages: Map
+ * }}
+ */
+function _mapStateToProps(state) {
+    return {
+        _transcriptMessages: state['features/subtitles'].transcriptMessages
+    };
+}
+export default connect(_mapStateToProps)(TranscriptionSubtitles);

react/features/subtitles/components/index.js

@@ -0,0 +1 @@
+export { default as TranscriptionSubtitles } from './TranscriptionSubtitles';

react/features/subtitles/index.js

@@ -0,0 +1,6 @@
+export * from './actions';
+export * from './actionTypes';
+export * from './components';
+
+import './middleware';
+import './reducer';

react/features/subtitles/middleware.js

@@ -0,0 +1,112 @@
+import { MiddlewareRegistry } from '../base/redux';
+
+import { ENDPOINT_MESSAGE_RECEIVED } from './actionTypes';
+import {
+    addTranscriptMessage,
+    removeTranscriptMessage,
+    updateTranscriptMessage
+} from './actions';
+
+const logger = require('jitsi-meet-logger').getLogger(__filename);
+
+/**
+* Time after which the rendered subtitles will be removed.
+*/
+const REMOVE_AFTER_MS = 3000;
+
+/**
+ * Middleware that catches actions related to transcript messages
+ * to be rendered in {@link TranscriptionSubtitles }
+ *
+ * @param {Store} store - Redux store.
+ * @returns {Function}
+ */
+MiddlewareRegistry.register(store => next => action => {
+
+    switch (action.type) {
+    case ENDPOINT_MESSAGE_RECEIVED:
+        return _endpointMessageReceived(store, next, action);
+    }
+
+    return next(action);
+});
+
+/**
+ * Notifies the feature transcription that the action
+ * {@code ENDPOINT_MESSAGE_RECEIVED} is being dispatched within a specific redux
+ * store.
+ *
+ * @param {Store} store - The redux store in which the specified {@code action}
+ * is being dispatched.
+ * @param {Dispatch} next - The redux {@code dispatch} function to
+ * dispatch the specified {@code action} to the specified {@code store}.
+ * @param {Action} action - The redux action {@code ENDPOINT_MESSAGE_RECEIVED}
+ * which is being dispatched in the specified {@code store}.
+ * @private
+ * @returns {Object} The value returned by {@code next(action)}.
+ */
+function _endpointMessageReceived({ dispatch, getState }, next, action) {
+    const json = action.json;
+
+    try {
+
+        // Let's first check if the given object has the correct
+        // type in the json, which identifies it as a json message sent
+        // from Jigasi with speech-to-to-text results
+        if (json.type === 'transcription-result') {
+            // Extract the useful data from the json.
+            const isInterim = json.is_interim;
+            const participantName = json.participant.name;
+            const stability = json.stability;
+            const text = json.transcript[0].text;
+            const transcriptMessageID = json.message_id;
+
+            // If this is the first result with the unique message ID,
+            // we add it to the state along with the name of the participant
+            // who said given text
+            if (!getState()['features/subtitles']
+                .transcriptMessages.has(transcriptMessageID)) {
+                dispatch(addTranscriptMessage(transcriptMessageID,
+                    participantName));
+            }
+            const { transcriptMessages } = getState()['features/subtitles'];
+            const newTranscriptMessage
+                = { ...transcriptMessages.get(transcriptMessageID) };
+
+            // If this is final result, update the state as a final result
+            // and start a count down to remove the subtitle from the state
+            if (!isInterim) {
+
+                newTranscriptMessage.final = text;
+                dispatch(updateTranscriptMessage(transcriptMessageID,
+                    newTranscriptMessage));
+
+                setTimeout(() => {
+                    dispatch(removeTranscriptMessage(transcriptMessageID));
+                }, REMOVE_AFTER_MS);
+            } else if (stability > 0.85) {
+
+                // If the message has a high stability, we can update the
+                // stable field of the state and remove the previously
+                // unstable results
+
+                newTranscriptMessage.stable = text;
+                newTranscriptMessage.unstable = undefined;
+                dispatch(updateTranscriptMessage(transcriptMessageID,
+                    newTranscriptMessage));
+            } else {
+                // Otherwise, this result has an unstable result, which we
+                // add to the state. The unstable result will be appended
+                // after the stable part.
+
+                newTranscriptMessage.unstable = text;
+                dispatch(updateTranscriptMessage(transcriptMessageID,
+                    newTranscriptMessage));
+            }
+        }
+    } catch (error) {
+        logger.error('Error occurred while updating transcriptions\n', error);
+    }
+
+    return next(action);
+}

react/features/subtitles/reducer.js

@@ -0,0 +1,99 @@
+import { ReducerRegistry } from '../base/redux';
+
+import {
+    ADD_TRANSCRIPT_MESSAGE,
+    REMOVE_TRANSCRIPT_MESSAGE,
+    UPDATE_TRANSCRIPT_MESSAGE
+} from './actionTypes';
+
+/**
+ * Default State for 'features/transcription' feature
+ */
+const defaultState = {
+    transcriptMessages: new Map()
+};
+
+/**
+ * Listen for actions for the transcription feature to be used by the actions
+ * to update the rendered transcription subtitles.
+ */
+ReducerRegistry.register('features/subtitles', (
+        state = defaultState, action) => {
+    switch (action.type) {
+    case ADD_TRANSCRIPT_MESSAGE:
+        return _addTranscriptMessage(state, action);
+
+    case REMOVE_TRANSCRIPT_MESSAGE:
+        return _removeTranscriptMessage(state, action);
+
+    case UPDATE_TRANSCRIPT_MESSAGE:
+        return _updateTranscriptMessage(state, action);
+    }
+
+    return state;
+});
+
+/**
+ * Reduces a specific Redux action ADD_TRANSCRIPT_MESSAGE of the feature
+ * transcription.
+ *
+ * @param {Object} state - The Redux state of the feature transcription.
+ * @param {Action} action -The Redux action ADD_TRANSCRIPT_MESSAGE to reduce.
+ * @returns {Object} The new state of the feature transcription after the
+ * reduction of the specified action.
+ */
+function _addTranscriptMessage(state,
+        { transcriptMessageID, participantName }) {
+    const newTranscriptMessages = new Map(state.transcriptMessages);
+
+    // Adds a new key,value pair to the Map once a new message arrives.
+    newTranscriptMessages.set(transcriptMessageID, { participantName });
+
+    return {
+        ...state,
+        transcriptMessages: newTranscriptMessages
+    };
+}
+
+/**
+ * Reduces a specific Redux action REMOVE_TRANSCRIPT_MESSAGE of the feature
+ * transcription.
+ *
+ * @param {Object} state - The Redux state of the feature transcription.
+ * @param {Action} action -The Redux action REMOVE_TRANSCRIPT_MESSAGE to reduce.
+ * @returns {Object} The new state of the feature transcription after the
+ * reduction of the specified action.
+ */
+function _removeTranscriptMessage(state, { transcriptMessageID }) {
+    const newTranscriptMessages = new Map(state.transcriptMessages);
+
+    // Deletes the key from Map once a final message arrives.
+    newTranscriptMessages.delete(transcriptMessageID);
+
+    return {
+        ...state,
+        transcriptMessages: newTranscriptMessages
+    };
+}
+
+/**
+ * Reduces a specific Redux action UPDATE_TRANSCRIPT_MESSAGE of the feature
+ * transcription.
+ *
+ * @param {Object} state - The Redux state of the feature transcription.
+ * @param {Action} action -The Redux action UPDATE_TRANSCRIPT_MESSAGE to reduce.
+ * @returns {Object} The new state of the feature transcription after the
+ * reduction of the specified action.
+ */
+function _updateTranscriptMessage(state,
+        { transcriptMessageID, newTranscriptMessage }) {
+    const newTranscriptMessages = new Map(state.transcriptMessages);
+
+    // Updates the new message for the given key in the Map.
+    newTranscriptMessages.set(transcriptMessageID, newTranscriptMessage);
+
+    return {
+        ...state,
+        transcriptMessages: newTranscriptMessages
+    };
+}