diff --git a/react/features/base/devices/services/index.js b/react/features/base/devices/services/index.js deleted file mode 100644 index 7849deaeb..000000000 --- a/react/features/base/devices/services/index.js +++ /dev/null @@ -1,2 +0,0 @@ -export * from './vad-reporter/Events'; -export * from './vad-reporter/VADReportingService'; diff --git a/react/features/base/devices/services/vad-reporter/Events.js b/react/features/base/devices/services/vad-reporter/Events.js deleted file mode 100644 index da110cfde..000000000 --- a/react/features/base/devices/services/vad-reporter/Events.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Event generated by VADReportingService when if finishes creating a VAD report for the monitored devices. - * The generated objects are of type Array, one score for each monitored device. - * @event VAD_REPORT_PUBLISHED - * @type Array with the following structure: - * @property {Date} timestamp - Timestamp at which the compute took place. - * @property {number} avgVAD - Average VAD score over monitored period of time. - * @property {string} deviceId - Associate local audio device ID. - */ -export const VAD_REPORT_PUBLISHED = 'vad-report-published'; diff --git a/react/features/base/devices/services/vad-reporter/VADReportingService.js b/react/features/base/devices/services/vad-reporter/VADReportingService.js deleted file mode 100644 index e1fde360f..000000000 --- a/react/features/base/devices/services/vad-reporter/VADReportingService.js +++ /dev/null @@ -1,284 +0,0 @@ -// @flow - -import { createRnnoiseProcessorPromise } from '../../../../rnnoise'; -import EventEmitter from 'events'; -import logger from '../../logger'; -import JitsiMeetJS, { JitsiDetectionEvents } from '../../../lib-jitsi-meet'; -import { VAD_REPORT_PUBLISHED } from './Events'; - -/** - * Sample rate used by TrackVADEmitter, this value determines how often the ScriptProcessorNode is going to call the - * process audio function and with what sample size. - * Basically lower values mean more callbacks with lower processing times bigger values less callbacks with longer - * processing times. This value is somewhere in the middle, so we strike a balance between flooding with callbacks - * and processing time. Possible values 256, 512, 1024, 2048, 4096, 8192, 16384. Passing other values will default - * to closes neighbor. - */ -const SCRIPT_NODE_SAMPLE_RATE = 4096; - -/** - * Context that contains the emitter and additional information about the device. - */ -type VADDeviceContext = { - - /** - * MediaDeviceInfo for associated context - */ - deviceInfo: MediaDeviceInfo, - - /** - * Array with VAD scores publish from the emitter. - */ - scoreArray: Array, - - /** - * TrackVADEmitter associated with media device - */ - vadEmitter: Object -}; - -/** - * Voice activity detection reporting service. The service create TrackVADEmitters for the provided devices and - * publishes an average of their VAD score over the specified interval via EventEmitter. - * The service is not reusable if destroyed a new one needs to be created, i.e. when a new device is added to the system - * a new service needs to be created and the old discarded. - */ -export default class VADReportingService extends EventEmitter { - /** - * Map containing context for devices currently being monitored by the reporting service. - */ - _contextMap: Map; - - /** - * State flag, check if the instance was destroyed. - */ - _destroyed: boolean = false; - - /** - * Delay at which to publish VAD score for monitored devices. - */ - _intervalDelay: number; - - /** - * Identifier for the interval publishing stats on the set interval. - */ - _intervalId: ?IntervalID; - - /** - * Constructor. - * - * @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices. - * @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score. - */ - constructor(intervalDelay: number) { - super(); - this._contextMap = new Map(); - this._intervalDelay = intervalDelay; - - logger.log(`Constructed VADReportingService with publish interval of: ${intervalDelay}`); - } - - /** - * Factory methods that creates the TrackVADEmitters for the associated array of devices and instantiates - * a VADReportingService. - * - * @param {Array} micDeviceList - Device list that is monitored inside the service. - * @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices. - * @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score. - * - * @returns {Promise} - */ - static async create(micDeviceList: Array, intervalDelay: number) { - const vadReportingService = new VADReportingService(intervalDelay); - const emitterPromiseArray = []; - - // Create a TrackVADEmitter for each provided audioinput device. - for (const micDevice of micDeviceList) { - if (micDevice.kind !== 'audioinput') { - logger.warn(`Provided device ${micDevice.label} -> ${micDevice.deviceId}, is not audioinput ignoring!`); - - return; - } - - logger.log(`Initializing VAD context for mic: ${micDevice.label} -> ${micDevice.deviceId}`); - - const rnnoiseProcessor = await createRnnoiseProcessorPromise(); - - const emitterPromise = JitsiMeetJS.createTrackVADEmitter( - micDevice.deviceId, - SCRIPT_NODE_SAMPLE_RATE, - rnnoiseProcessor - ).then(emitter => { - emitter.on( - JitsiDetectionEvents.VAD_SCORE_PUBLISHED, - vadReportingService._devicePublishVADScore.bind(vadReportingService) - ); - - return { - vadEmitter: emitter, - deviceInfo: micDevice, - scoreArray: [] - }; - }); - - emitterPromiseArray.push(emitterPromise); - } - - // Once all the TrackVADEmitter promises are resolved check if all of them resolved properly if not reject - // the promise and clear the already created emitters. - // $FlowFixMe - allSettled is not part of flow prototype even though it's a valid Promise function - return Promise.allSettled(emitterPromiseArray).then(outcomeArray => { - const vadContextArray = []; - const rejectedEmitterPromiseArray = []; - - for (const outcome of outcomeArray) { - if (outcome.status === 'fulfilled') { - vadContextArray.push(outcome.value); - } else { - // Promise was rejected. - logger.error(`Create TrackVADEmitter promise failed with ${outcome.reason}`); - - rejectedEmitterPromiseArray.push(outcome); - } - } - - // Check if there were any rejected promises and clear the already created ones list. - if (rejectedEmitterPromiseArray.length > 0) { - logger.error('Cleaning up remaining VADDeviceContext, due to create fail!'); - - for (const context of vadContextArray) { - context.vadEmitter.destroy(); - } - - // Reject create promise if one emitter failed to instantiate, we might one just ignore it, - // leaving it like this for now - throw new Error('Create VADReportingService failed due to TrackVADEmitter creation issues!'); - } - - vadReportingService._setVADContextArray(vadContextArray); - vadReportingService._startPublish(); - - return vadReportingService; - }); - } - - /** - * Destroy TrackVADEmitters and clear the context map. - * - * @returns {void} - */ - _clearContextMap() { - for (const vadContext of this._contextMap.values()) { - vadContext.vadEmitter.destroy(); - } - this._contextMap.clear(); - } - - /** - * Set the watched device contexts. - * - * @param {Array} vadContextArray - List of mics. - * @returns {void} - */ - _setVADContextArray(vadContextArray: Array): void { - for (const vadContext of vadContextArray) { - this._contextMap.set(vadContext.deviceInfo.deviceId, vadContext); - } - } - - /** - * Start the setInterval reporting process. - * - * @returns {void}. - */ - _startPublish() { - logger.log('VADReportingService started publishing.'); - this._intervalId = setInterval(() => { - this._reportVadScore(); - }, this._intervalDelay); - } - - /** - * Function called at set interval with selected compute. The result will be published on the set callback. - * - * @returns {void} - * @fires VAD_REPORT_PUBLISHED - */ - _reportVadScore() { - const vadComputeScoreArray = []; - const computeTimestamp = Date.now(); - - // Go through each device and compute cumulated VAD score. - - for (const [ deviceId, vadContext ] of this._contextMap) { - const nrOfVADScores = vadContext.scoreArray.length; - let vadSum = 0; - - vadContext.scoreArray.forEach(vadScore => { - vadSum += vadScore.score; - }); - - // TODO For now we just calculate the average score for each device, more compute algorithms will be added. - const avgVAD = vadSum / nrOfVADScores; - - vadContext.scoreArray = []; - - vadComputeScoreArray.push({ - timestamp: computeTimestamp, - score: avgVAD, - deviceId - }); - } - - /** - * Once the computation for all the tracked devices is done, fire an event containing all the necessary - * information. - * - * @event VAD_REPORT_PUBLISHED - * @type Array with the following structure: - * @property {Date} timestamp - Timestamo at which the compute took place. - * @property {number} avgVAD - Average VAD score over monitored period of time. - * @property {string} deviceId - Associate local audio device ID. - */ - this.emit(VAD_REPORT_PUBLISHED, vadComputeScoreArray); - } - - /** - * Callback method passed to vad emitters in order to publish their score. - * - * @param {Object} vadScore -VAD score emitted by. - * @param {Date} vadScore.timestamp - Exact time at which processed PCM sample was generated. - * @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7). - * @param {string} vadScore.deviceId - Device id of the associated track. - * @returns {void} - * @listens VAD_SCORE_PUBLISHED - */ - _devicePublishVADScore(vadScore: Object) { - const context = this._contextMap.get(vadScore.deviceId); - - if (context) { - context.scoreArray.push(vadScore); - } - } - - /** - * Destroy the VADReportingService, stops the setInterval reporting, destroys the emitters and clears the map. - * After this call the instance is no longer usable. - * - * @returns {void}. - */ - destroy() { - if (this._destroyed) { - return; - } - - logger.log('Destroying VADReportingService.'); - - if (this._intervalId) { - clearInterval(this._intervalId); - this._intervalId = null; - } - this._clearContextMap(); - this._destroyed = true; - } -} diff --git a/react/features/base/tracks/actionTypes.js b/react/features/base/tracks/actionTypes.js index 759c7075c..e59a8cdb9 100644 --- a/react/features/base/tracks/actionTypes.js +++ b/react/features/base/tracks/actionTypes.js @@ -3,11 +3,11 @@ * no data from source notification. Used to check if such a notification was previously displayed. * * { - * type: SET_NO_SRC_DATA_NOTI_UID, + * type: SET_NO_SRC_DATA_NOTIFICATION_UID, * uid: ?number * } */ -export const SET_NO_SRC_DATA_NOTI_UID = 'SET_NO_SRC_DATA_NOTI_UID'; +export const SET_NO_SRC_DATA_NOTIFICATION_UID = 'SET_NO_SRC_DATA_NOTIFICATION_UID'; /** * The type of redux action dispatched to disable screensharing or to start the diff --git a/react/features/base/tracks/reducer.js b/react/features/base/tracks/reducer.js index df2a28402..ec678f322 100644 --- a/react/features/base/tracks/reducer.js +++ b/react/features/base/tracks/reducer.js @@ -2,7 +2,7 @@ import { PARTICIPANT_ID_CHANGED } from '../participants'; import { ReducerRegistry, set } from '../redux'; import { - SET_NO_SRC_DATA_NOTI_UID, + SET_NO_SRC_DATA_NOTIFICATION_UID, TRACK_ADDED, TRACK_CREATE_CANCELED, TRACK_CREATE_ERROR, @@ -140,7 +140,7 @@ ReducerRegistry.register('features/base/tracks', (state = [], action) => { */ ReducerRegistry.register('features/base/no-src-data', (state = {}, action) => { switch (action.type) { - case SET_NO_SRC_DATA_NOTI_UID: + case SET_NO_SRC_DATA_NOTIFICATION_UID: return set(state, 'noSrcDataNotificationUid', action.uid); default: diff --git a/react/features/no-audio-signal/middleware.js b/react/features/no-audio-signal/middleware.js index 075f0752f..5d0b4b8a7 100644 --- a/react/features/no-audio-signal/middleware.js +++ b/react/features/no-audio-signal/middleware.js @@ -1,5 +1,7 @@ // @flow import { setNoAudioSignalNotificationUid } from './actions'; +import { NO_AUDIO_SIGNAL_SOUND_ID } from './constants'; +import { NO_AUDIO_SIGNAL_SOUND_FILE } from './sounds'; import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app'; import { CONFERENCE_JOINED } from '../base/conference'; import { @@ -10,9 +12,7 @@ import JitsiMeetJS, { JitsiConferenceEvents } from '../base/lib-jitsi-meet'; import { MiddlewareRegistry } from '../base/redux'; import { updateSettings } from '../base/settings'; import { playSound, registerSound, unregisterSound } from '../base/sounds'; -import { NO_AUDIO_SIGNAL_SOUND_ID } from './constants'; import { hideNotification, showNotification } from '../notifications'; -import { NO_AUDIO_SIGNAL_SOUND_FILE } from './sounds'; MiddlewareRegistry.register(store => next => async action => { const result = next(action); @@ -34,6 +34,7 @@ MiddlewareRegistry.register(store => next => async action => { confAudioInputState = hasAudioInput; + // In case the notification is displayed but the conference detected audio input signal we hide it. if (noAudioSignalNotificationUid && hasAudioInput) { dispatch(hideNotification(noAudioSignalNotificationUid)); dispatch(setNoAudioSignalNotificationUid()); @@ -50,7 +51,7 @@ MiddlewareRegistry.register(store => next => async action => { } // Force the flag to false in case AUDIO_INPUT_STATE_CHANGE is received after the notification is displayed, - // thus making sure we check properly if the notification should display. + // possibly preventing the notification from displaying because of an outdated state. confAudioInputState = false;