Show subtitles when Jigasi sends transcription results in JSON (#1914)

* Show subtitles when Jigasi sends transcription results in JSON

* fix: Import PropTypes from prop-types.

* apply feedback on initial PR

* Changed Object to Map, alphabetic ordering fixes ,css changes in transcription subtitles

* Sends Map of transcriptMessages as prop to Component

* Documentation fixes and uses config in redux state

* Minor doc fix

* rename feature 'transcription' to 'subtitles'

* Moves subtitles config to interfaceConfig and minor fixes

* minor lint fix
This commit is contained in:
Nik 2018-07-17 19:31:12 +02:00 committed by bgrozev
parent 13ee67d15c
commit d3dd54ac3b
15 changed files with 456 additions and 3 deletions

View File

@ -109,6 +109,7 @@ import {
} from './react/features/overlay'; } from './react/features/overlay';
import { setSharedVideoStatus } from './react/features/shared-video'; import { setSharedVideoStatus } from './react/features/shared-video';
import { isButtonEnabled } from './react/features/toolbox'; import { isButtonEnabled } from './react/features/toolbox';
import { endpointMessageReceived } from './react/features/subtitles';
const logger = require('jitsi-meet-logger').getLogger(__filename); const logger = require('jitsi-meet-logger').getLogger(__filename);
@ -1875,6 +1876,10 @@ export default {
} }
); );
room.on(
JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
(...args) => APP.store.dispatch(endpointMessageReceived(...args)));
room.on( room.on(
JitsiConferenceEvents.LOCK_STATE_CHANGED, JitsiConferenceEvents.LOCK_STATE_CHANGED,
(...args) => APP.store.dispatch(lockStateChanged(room, ...args))); (...args) => APP.store.dispatch(lockStateChanged(room, ...args)));

View File

@ -252,7 +252,6 @@ var config = {
// maintenance at 01:00 AM GMT, // maintenance at 01:00 AM GMT,
// noticeMessage: '', // noticeMessage: '',
// Stats // Stats
// //

View File

@ -0,0 +1,13 @@
.transcription-subtitles{
bottom: 10%;
font-size: 16px;
font-weight: 1000;
opacity: 0.80;
position: absolute;
text-shadow: 0px 0px 1px rgba(0,0,0,0.3),
0px 1px 1px rgba(0,0,0,0.3),
1px 0px 1px rgba(0,0,0,0.3),
0px 0px 1px rgba(0,0,0,0.3);
width: 100%;
z-index: $zindex2;
}

View File

@ -77,5 +77,5 @@
@import 'unsupported-browser/main'; @import 'unsupported-browser/main';
@import 'modals/invite/add-people'; @import 'modals/invite/add-people';
@import 'deep-linking/main'; @import 'deep-linking/main';
@import 'transcription-subtitles';
/* Modules END */ /* Modules END */

View File

@ -80,6 +80,14 @@ var interfaceConfig = {
DISABLE_FOCUS_INDICATOR: false, DISABLE_FOCUS_INDICATOR: false,
DISABLE_DOMINANT_SPEAKER_INDICATOR: false, DISABLE_DOMINANT_SPEAKER_INDICATOR: false,
/**
* Whether the speech to text transcription subtitles panel is disabled.
* If {@code undefined}, defaults to {@code false}.
*
* @type {boolean}
*/
DISABLE_TRANSCRIPTION_SUBTITLES: false,
/** /**
* Whether the ringing sound in the call/ring overlay is disabled. If * Whether the ringing sound in the call/ring overlay is disabled. If
* {@code undefined}, defaults to {@code false}. * {@code undefined}, defaults to {@code false}.

View File

@ -54,7 +54,6 @@ import {
getCurrentConference, getCurrentConference,
sendLocalParticipant sendLocalParticipant
} from './functions'; } from './functions';
import type { Dispatch } from 'redux'; import type { Dispatch } from 'redux';
const logger = require('jitsi-meet-logger').getLogger(__filename); const logger = require('jitsi-meet-logger').getLogger(__filename);

View File

@ -4,6 +4,7 @@ import PropTypes from 'prop-types';
import React, { Component } from 'react'; import React, { Component } from 'react';
import { Watermarks } from '../../base/react'; import { Watermarks } from '../../base/react';
import { TranscriptionSubtitles } from '../../subtitles/';
import Labels from './Labels'; import Labels from './Labels';
@ -70,6 +71,8 @@ export default class LargeVideo extends Component<*> {
muted = { true } /> muted = { true } />
</div> </div>
</div> </div>
{ interfaceConfig.DISABLE_TRANSCRIPTION_SUBTITLES
? null : <TranscriptionSubtitles /> }
<span id = 'localConnectionMessage' /> <span id = 'localConnectionMessage' />
{ this.props.hideVideoQualityLabel { this.props.hideVideoQualityLabel
? null : <Labels /> } ? null : <Labels /> }

View File

@ -0,0 +1,46 @@
/**
* The type of (redux) action which indicates that a transcript with
* a new message_id is received.
*
* {
* type: ADD_TRANSCRIPT_MESSAGE,
* transcriptMessageID: string,
* participantName: string
* }
*/
export const ADD_TRANSCRIPT_MESSAGE = Symbol('ADD_TRANSCRIPT_MESSAGE');
/**
* The type of (redux) action which indicates that an endpoint message
* sent by another participant to the data channel is received.
*
* {
* type: ENDPOINT_MESSAGE_RECEIVED,
* participant: Object,
* json: Object
* }
*/
export const ENDPOINT_MESSAGE_RECEIVED = Symbol('ENDPOINT_MESSAGE_RECEIVED');
/**
* The type of (redux) action which indicates that an existing transcript
* has to be removed from the state.
*
* {
* type: REMOVE_TRANSCRIPT_MESSAGE,
* transciptMessageID: string,
* }
*/
export const REMOVE_TRANSCRIPT_MESSAGE = Symbol('REMOVE_TRANSCRIPT_MESSAGE');
/**
* The type of (redux) action which indicates that a transcript with an
* existing message_id to be updated is received.
*
* {
* type: UPDATE_TRANSCRIPT_MESSAGE,
* transcriptMessageID: string,
* newTranscriptMessage: Object
* }
*/
export const UPDATE_TRANSCRIPT_MESSAGE = Symbol('UPDATE_TRANSCRIPT_MESSAGE');

View File

@ -0,0 +1,84 @@
// @flow
import {
ADD_TRANSCRIPT_MESSAGE,
ENDPOINT_MESSAGE_RECEIVED,
REMOVE_TRANSCRIPT_MESSAGE,
UPDATE_TRANSCRIPT_MESSAGE
} from './actionTypes';
/**
* Signals that a transcript with a new message_id is received.
*
* @param {string} transcriptMessageID - The new message_id.
* @param {string} participantName - The participant name of the sender.
* @returns {{
* type: ADD_TRANSCRIPT_MESSAGE,
* transcriptMessageID: string,
* participantName: string
* }}
*/
export function addTranscriptMessage(transcriptMessageID: string,
participantName: string) {
return {
type: ADD_TRANSCRIPT_MESSAGE,
transcriptMessageID,
participantName
};
}
/**
* Signals that a participant sent an endpoint message on the data channel.
*
* @param {Object} participant - The participant details sending the message.
* @param {Object} json - The json carried by the endpoint message.
* @returns {{
* type: ENDPOINT_MESSAGE_RECEIVED,
* participant: Object,
* json: Object
* }}
*/
export function endpointMessageReceived(participant: Object, json: Object) {
return {
type: ENDPOINT_MESSAGE_RECEIVED,
participant,
json
};
}
/**
* Signals that a transcript has to be removed from the state.
*
* @param {string} transcriptMessageID - The message_id to be removed.
* @returns {{
* type: REMOVE_TRANSCRIPT_MESSAGE,
* transcriptMessageID: string,
* }}
*/
export function removeTranscriptMessage(transcriptMessageID: string) {
return {
type: REMOVE_TRANSCRIPT_MESSAGE,
transcriptMessageID
};
}
/**
* Signals that a transcript with an existing message_id to be updated
* is received.
*
* @param {string} transcriptMessageID -The transcript message_id to be updated.
* @param {Object} newTranscriptMessage - The updated transcript message.
* @returns {{
* type: UPDATE_TRANSCRIPT_MESSAGE,
* transcriptMessageID: string,
* newTranscriptMessage: Object
* }}
*/
export function updateTranscriptMessage(transcriptMessageID: string,
newTranscriptMessage: Object) {
return {
type: UPDATE_TRANSCRIPT_MESSAGE,
transcriptMessageID,
newTranscriptMessage
};
}

View File

@ -0,0 +1,78 @@
// @flow
import React, { Component } from 'react';
import { connect } from 'react-redux';
/**
* The type of the React {@code Component} props of
* {@link TranscriptionSubtitles}.
*/
type Props = {
/**
* Map of transcriptMessageID's with corresponding transcriptMessage.
*/
_transcriptMessages: Map<string, Object>
};
/**
* React {@code Component} which can display speech-to-text results from
* Jigasi as subtitles.
*/
class TranscriptionSubtitles extends Component<Props> {
/**
* Implements React's {@link Component#render()}.
*
* @inheritdoc
* @returns {ReactElement}
*/
render() {
const paragraphs = [];
for (const [ transcriptMessageID, transcriptMessage ]
of this.props._transcriptMessages) {
let text;
if (transcriptMessage) {
text = `${transcriptMessage.participantName}: `;
if (transcriptMessage.final) {
text += transcriptMessage.final;
} else {
const stable = transcriptMessage.stable || '';
const unstable = transcriptMessage.unstable || '';
text += stable + unstable;
}
paragraphs.push(
<p key = { transcriptMessageID }> { text } </p>
);
}
}
return (
<div className = 'transcription-subtitles' >
{ paragraphs }
</div>
);
}
}
/**
* Maps the transcriptionSubtitles in the Redux state to the associated
* props of {@code TranscriptionSubtitles}.
*
* @param {Object} state - The Redux state.
* @private
* @returns {{
* _transcriptMessages: Map
* }}
*/
function _mapStateToProps(state) {
return {
_transcriptMessages: state['features/subtitles'].transcriptMessages
};
}
export default connect(_mapStateToProps)(TranscriptionSubtitles);

View File

@ -0,0 +1 @@
export { default as TranscriptionSubtitles } from './TranscriptionSubtitles';

View File

@ -0,0 +1,6 @@
export * from './actions';
export * from './actionTypes';
export * from './components';
import './middleware';
import './reducer';

View File

@ -0,0 +1,112 @@
import { MiddlewareRegistry } from '../base/redux';
import { ENDPOINT_MESSAGE_RECEIVED } from './actionTypes';
import {
addTranscriptMessage,
removeTranscriptMessage,
updateTranscriptMessage
} from './actions';
const logger = require('jitsi-meet-logger').getLogger(__filename);
/**
* Time after which the rendered subtitles will be removed.
*/
const REMOVE_AFTER_MS = 3000;
/**
* Middleware that catches actions related to transcript messages
* to be rendered in {@link TranscriptionSubtitles }
*
* @param {Store} store - Redux store.
* @returns {Function}
*/
MiddlewareRegistry.register(store => next => action => {
switch (action.type) {
case ENDPOINT_MESSAGE_RECEIVED:
return _endpointMessageReceived(store, next, action);
}
return next(action);
});
/**
* Notifies the feature transcription that the action
* {@code ENDPOINT_MESSAGE_RECEIVED} is being dispatched within a specific redux
* store.
*
* @param {Store} store - The redux store in which the specified {@code action}
* is being dispatched.
* @param {Dispatch} next - The redux {@code dispatch} function to
* dispatch the specified {@code action} to the specified {@code store}.
* @param {Action} action - The redux action {@code ENDPOINT_MESSAGE_RECEIVED}
* which is being dispatched in the specified {@code store}.
* @private
* @returns {Object} The value returned by {@code next(action)}.
*/
function _endpointMessageReceived({ dispatch, getState }, next, action) {
const json = action.json;
try {
// Let's first check if the given object has the correct
// type in the json, which identifies it as a json message sent
// from Jigasi with speech-to-to-text results
if (json.type === 'transcription-result') {
// Extract the useful data from the json.
const isInterim = json.is_interim;
const participantName = json.participant.name;
const stability = json.stability;
const text = json.transcript[0].text;
const transcriptMessageID = json.message_id;
// If this is the first result with the unique message ID,
// we add it to the state along with the name of the participant
// who said given text
if (!getState()['features/subtitles']
.transcriptMessages.has(transcriptMessageID)) {
dispatch(addTranscriptMessage(transcriptMessageID,
participantName));
}
const { transcriptMessages } = getState()['features/subtitles'];
const newTranscriptMessage
= { ...transcriptMessages.get(transcriptMessageID) };
// If this is final result, update the state as a final result
// and start a count down to remove the subtitle from the state
if (!isInterim) {
newTranscriptMessage.final = text;
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
setTimeout(() => {
dispatch(removeTranscriptMessage(transcriptMessageID));
}, REMOVE_AFTER_MS);
} else if (stability > 0.85) {
// If the message has a high stability, we can update the
// stable field of the state and remove the previously
// unstable results
newTranscriptMessage.stable = text;
newTranscriptMessage.unstable = undefined;
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
} else {
// Otherwise, this result has an unstable result, which we
// add to the state. The unstable result will be appended
// after the stable part.
newTranscriptMessage.unstable = text;
dispatch(updateTranscriptMessage(transcriptMessageID,
newTranscriptMessage));
}
}
} catch (error) {
logger.error('Error occurred while updating transcriptions\n', error);
}
return next(action);
}

View File

@ -0,0 +1,99 @@
import { ReducerRegistry } from '../base/redux';
import {
ADD_TRANSCRIPT_MESSAGE,
REMOVE_TRANSCRIPT_MESSAGE,
UPDATE_TRANSCRIPT_MESSAGE
} from './actionTypes';
/**
* Default State for 'features/transcription' feature
*/
const defaultState = {
transcriptMessages: new Map()
};
/**
* Listen for actions for the transcription feature to be used by the actions
* to update the rendered transcription subtitles.
*/
ReducerRegistry.register('features/subtitles', (
state = defaultState, action) => {
switch (action.type) {
case ADD_TRANSCRIPT_MESSAGE:
return _addTranscriptMessage(state, action);
case REMOVE_TRANSCRIPT_MESSAGE:
return _removeTranscriptMessage(state, action);
case UPDATE_TRANSCRIPT_MESSAGE:
return _updateTranscriptMessage(state, action);
}
return state;
});
/**
* Reduces a specific Redux action ADD_TRANSCRIPT_MESSAGE of the feature
* transcription.
*
* @param {Object} state - The Redux state of the feature transcription.
* @param {Action} action -The Redux action ADD_TRANSCRIPT_MESSAGE to reduce.
* @returns {Object} The new state of the feature transcription after the
* reduction of the specified action.
*/
function _addTranscriptMessage(state,
{ transcriptMessageID, participantName }) {
const newTranscriptMessages = new Map(state.transcriptMessages);
// Adds a new key,value pair to the Map once a new message arrives.
newTranscriptMessages.set(transcriptMessageID, { participantName });
return {
...state,
transcriptMessages: newTranscriptMessages
};
}
/**
* Reduces a specific Redux action REMOVE_TRANSCRIPT_MESSAGE of the feature
* transcription.
*
* @param {Object} state - The Redux state of the feature transcription.
* @param {Action} action -The Redux action REMOVE_TRANSCRIPT_MESSAGE to reduce.
* @returns {Object} The new state of the feature transcription after the
* reduction of the specified action.
*/
function _removeTranscriptMessage(state, { transcriptMessageID }) {
const newTranscriptMessages = new Map(state.transcriptMessages);
// Deletes the key from Map once a final message arrives.
newTranscriptMessages.delete(transcriptMessageID);
return {
...state,
transcriptMessages: newTranscriptMessages
};
}
/**
* Reduces a specific Redux action UPDATE_TRANSCRIPT_MESSAGE of the feature
* transcription.
*
* @param {Object} state - The Redux state of the feature transcription.
* @param {Action} action -The Redux action UPDATE_TRANSCRIPT_MESSAGE to reduce.
* @returns {Object} The new state of the feature transcription after the
* reduction of the specified action.
*/
function _updateTranscriptMessage(state,
{ transcriptMessageID, newTranscriptMessage }) {
const newTranscriptMessages = new Map(state.transcriptMessages);
// Updates the new message for the given key in the Map.
newTranscriptMessages.set(transcriptMessageID, newTranscriptMessage);
return {
...state,
transcriptMessages: newTranscriptMessages
};
}