From 81853d971a430c0257accbc5d5c4e42795a781eb Mon Sep 17 00:00:00 2001 From: Praveen Gupta Date: Mon, 6 Aug 2018 09:24:37 +0000 Subject: [PATCH] [WEB] Show final translated speech to text results as subtitles (#3276) * Shows final translated speech to text results as subtitles * Use conference from redux state and removes addTranscriptMessage --- react/features/subtitles/actionTypes.js | 13 +---- react/features/subtitles/actions.js | 23 +------- react/features/subtitles/middleware.js | 76 ++++++++++++++++++------- react/features/subtitles/reducer.js | 26 --------- 4 files changed, 58 insertions(+), 80 deletions(-) diff --git a/react/features/subtitles/actionTypes.js b/react/features/subtitles/actionTypes.js index db8509bb7..c535b7f72 100644 --- a/react/features/subtitles/actionTypes.js +++ b/react/features/subtitles/actionTypes.js @@ -1,14 +1,3 @@ -/** - * The type of (redux) action which indicates that a transcript with - * a new message_id is received. - * - * { - * type: ADD_TRANSCRIPT_MESSAGE, - * transcriptMessageID: string, - * participantName: string - * } - */ -export const ADD_TRANSCRIPT_MESSAGE = Symbol('ADD_TRANSCRIPT_MESSAGE'); /** * The type of (redux) action which indicates that an endpoint message @@ -35,7 +24,7 @@ export const REMOVE_TRANSCRIPT_MESSAGE = Symbol('REMOVE_TRANSCRIPT_MESSAGE'); /** * The type of (redux) action which indicates that a transcript with an - * existing message_id to be updated is received. + * given message_id to be added or updated is received. * * { * type: UPDATE_TRANSCRIPT_MESSAGE, diff --git a/react/features/subtitles/actions.js b/react/features/subtitles/actions.js index 461bdf50b..5a8d629cf 100644 --- a/react/features/subtitles/actions.js +++ b/react/features/subtitles/actions.js @@ -1,32 +1,11 @@ // @flow import { - ADD_TRANSCRIPT_MESSAGE, ENDPOINT_MESSAGE_RECEIVED, REMOVE_TRANSCRIPT_MESSAGE, UPDATE_TRANSCRIPT_MESSAGE } from './actionTypes'; -/** - * Signals that a transcript with a new message_id is received. - * - * @param {string} transcriptMessageID - The new message_id. - * @param {string} participantName - The participant name of the sender. - * @returns {{ - * type: ADD_TRANSCRIPT_MESSAGE, - * transcriptMessageID: string, - * participantName: string - * }} - */ -export function addTranscriptMessage(transcriptMessageID: string, - participantName: string) { - return { - type: ADD_TRANSCRIPT_MESSAGE, - transcriptMessageID, - participantName - }; -} - /** * Signals that a participant sent an endpoint message on the data channel. * @@ -63,7 +42,7 @@ export function removeTranscriptMessage(transcriptMessageID: string) { } /** - * Signals that a transcript with an existing message_id to be updated + * Signals that a transcript with the given message_id to be added or updated * is received. * * @param {string} transcriptMessageID -The transcript message_id to be updated. diff --git a/react/features/subtitles/middleware.js b/react/features/subtitles/middleware.js index 6f687b186..e4e3399ce 100644 --- a/react/features/subtitles/middleware.js +++ b/react/features/subtitles/middleware.js @@ -1,14 +1,35 @@ +// @flow + import { MiddlewareRegistry } from '../base/redux'; import { ENDPOINT_MESSAGE_RECEIVED } from './actionTypes'; import { - addTranscriptMessage, removeTranscriptMessage, updateTranscriptMessage } from './actions'; const logger = require('jitsi-meet-logger').getLogger(__filename); +declare var APP: Object; + +/** + * The type of json-message which indicates that json carries a + * transcription result. + */ +const JSON_TYPE_TRANSCRIPTION_RESULT = 'transcription-result'; + +/** + * The type of json-message which indicates that json carries a + * translation result. + */ +const JSON_TYPE_TRANSLATION_RESULT = 'translation-result'; + +/** + * The local participant property which is used to store the language + * preference for translation for a participant. + */ +const P_NAME_TRANSLATION_LANGUAGE = 'translation_language'; + /** * Time after which the rendered subtitles will be removed. */ @@ -47,31 +68,46 @@ MiddlewareRegistry.register(store => next => action => { */ function _endpointMessageReceived({ dispatch, getState }, next, action) { const json = action.json; + const translationLanguage + = getState()['features/base/conference'].conference + .getLocalParticipantProperty(P_NAME_TRANSLATION_LANGUAGE); try { + const transcriptMessageID = json.message_id; + const participantName = json.participant.name; + const isInterim = json.is_interim; + const stability = json.stability; + + if (json.type === JSON_TYPE_TRANSLATION_RESULT + && json.language === translationLanguage) { + // Displays final results in the target language if translation is + // enabled. + + const newTranscriptMessage = { + participantName, + final: json.text + }; + + dispatch(updateTranscriptMessage(transcriptMessageID, + newTranscriptMessage)); + + setTimeout(() => { + dispatch(removeTranscriptMessage(transcriptMessageID)); + }, REMOVE_AFTER_MS); + + } else if (json.type === JSON_TYPE_TRANSCRIPTION_RESULT + && !translationLanguage) { + // Displays interim and final results without any translation if + // translations are disabled. - // Let's first check if the given object has the correct - // type in the json, which identifies it as a json message sent - // from Jigasi with speech-to-to-text results - if (json.type === 'transcription-result') { - // Extract the useful data from the json. - const isInterim = json.is_interim; - const participantName = json.participant.name; - const stability = json.stability; const text = json.transcript[0].text; - const transcriptMessageID = json.message_id; - // If this is the first result with the unique message ID, - // we add it to the state along with the name of the participant - // who said given text - if (!getState()['features/subtitles'] - .transcriptMessages.has(transcriptMessageID)) { - dispatch(addTranscriptMessage(transcriptMessageID, - participantName)); - } - const { transcriptMessages } = getState()['features/subtitles']; + // We update the previous transcript message with the same + // message ID or adds a new transcript message if it does not + // exist in the map. const newTranscriptMessage - = { ...transcriptMessages.get(transcriptMessageID) }; + = { ...getState()['features/subtitles'].transcriptMessages + .get(transcriptMessageID) || { participantName } }; // If this is final result, update the state as a final result // and start a count down to remove the subtitle from the state diff --git a/react/features/subtitles/reducer.js b/react/features/subtitles/reducer.js index 69a9f3ba7..c1cdd9b5a 100644 --- a/react/features/subtitles/reducer.js +++ b/react/features/subtitles/reducer.js @@ -1,7 +1,6 @@ import { ReducerRegistry } from '../base/redux'; import { - ADD_TRANSCRIPT_MESSAGE, REMOVE_TRANSCRIPT_MESSAGE, UPDATE_TRANSCRIPT_MESSAGE } from './actionTypes'; @@ -20,9 +19,6 @@ const defaultState = { ReducerRegistry.register('features/subtitles', ( state = defaultState, action) => { switch (action.type) { - case ADD_TRANSCRIPT_MESSAGE: - return _addTranscriptMessage(state, action); - case REMOVE_TRANSCRIPT_MESSAGE: return _removeTranscriptMessage(state, action); @@ -33,28 +29,6 @@ ReducerRegistry.register('features/subtitles', ( return state; }); -/** - * Reduces a specific Redux action ADD_TRANSCRIPT_MESSAGE of the feature - * transcription. - * - * @param {Object} state - The Redux state of the feature transcription. - * @param {Action} action -The Redux action ADD_TRANSCRIPT_MESSAGE to reduce. - * @returns {Object} The new state of the feature transcription after the - * reduction of the specified action. - */ -function _addTranscriptMessage(state, - { transcriptMessageID, participantName }) { - const newTranscriptMessages = new Map(state.transcriptMessages); - - // Adds a new key,value pair to the Map once a new message arrives. - newTranscriptMessages.set(transcriptMessageID, { participantName }); - - return { - ...state, - transcriptMessages: newTranscriptMessages - }; -} - /** * Reduces a specific Redux action REMOVE_TRANSCRIPT_MESSAGE of the feature * transcription.