haskal
/
jiti-meet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

setNoSrcDataNotificationUid

This commit is contained in:
Andrei Gavrilescu 2019-11-25 14:31:08 +02:00 committed by Hristo Terezov
parent 191da551e3
commit 7828bf8d46
6 changed files with 8 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
react/features/base/devices/services/index.js
View File

@ -1,2 +0,0 @@
export * from './vad-reporter/Events';
export * from './vad-reporter/VADReportingService';

10
react/features/base/devices/services/vad-reporter/Events.js
View File

@ -1,10 +0,0 @@
/**
* Event generated by VADReportingService when if finishes creating a VAD report for the monitored devices.
* The generated objects are of type Array<Object>, one score for each monitored device.
* @event VAD_REPORT_PUBLISHED
* @type Array<Object> with the following structure:
* @property {Date} timestamp - Timestamp at which the compute took place.
* @property {number} avgVAD - Average VAD score over monitored period of time.
* @property {string} deviceId - Associate local audio device ID.
*/
export const VAD_REPORT_PUBLISHED = 'vad-report-published';

284
react/features/base/devices/services/vad-reporter/VADReportingService.js
View File

@ -1,284 +0,0 @@
// @flow
import { createRnnoiseProcessorPromise } from '../../../../rnnoise';
import EventEmitter from 'events';
import logger from '../../logger';
import JitsiMeetJS, { JitsiDetectionEvents } from '../../../lib-jitsi-meet';
import { VAD_REPORT_PUBLISHED } from './Events';
/**
* Sample rate used by TrackVADEmitter, this value determines how often the ScriptProcessorNode is going to call the
* process audio function and with what sample size.
* Basically lower values mean more callbacks with lower processing times bigger values less callbacks with longer
* processing times. This value is somewhere in the middle, so we strike a balance between flooding with callbacks
* and processing time. Possible values 256, 512, 1024, 2048, 4096, 8192, 16384. Passing other values will default
* to closes neighbor.
*/
const SCRIPT_NODE_SAMPLE_RATE = 4096;
/**
* Context that contains the emitter and additional information about the device.
*/
type VADDeviceContext = {
/**
* MediaDeviceInfo for associated context
*/
deviceInfo: MediaDeviceInfo,
/**
* Array with VAD scores publish from the emitter.
*/
scoreArray: Array<Object>,
/**
* TrackVADEmitter associated with media device
*/
vadEmitter: Object
};
/**
* Voice activity detection reporting service. The service create TrackVADEmitters for the provided devices and
* publishes an average of their VAD score over the specified interval via EventEmitter.
* The service is not reusable if destroyed a new one needs to be created, i.e. when a new device is added to the system
* a new service needs to be created and the old discarded.
*/
export default class VADReportingService extends EventEmitter {
/**
* Map containing context for devices currently being monitored by the reporting service.
*/
_contextMap: Map<string, VADDeviceContext>;
/**
* State flag, check if the instance was destroyed.
*/
_destroyed: boolean = false;
/**
* Delay at which to publish VAD score for monitored devices.
*/
_intervalDelay: number;
/**
* Identifier for the interval publishing stats on the set interval.
*/
_intervalId: ?IntervalID;
/**
* Constructor.
*
* @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices.
* @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score.
*/
constructor(intervalDelay: number) {
super();
this._contextMap = new Map();
this._intervalDelay = intervalDelay;
logger.log(`Constructed VADReportingService with publish interval of: ${intervalDelay}`);
}
/**
* Factory methods that creates the TrackVADEmitters for the associated array of devices and instantiates
* a VADReportingService.
*
* @param {Array<MediaDeviceInfo>} micDeviceList - Device list that is monitored inside the service.
* @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices.
* @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score.
*
* @returns {Promise<VADReportingService>}
*/
static async create(micDeviceList: Array<MediaDeviceInfo>, intervalDelay: number) {
const vadReportingService = new VADReportingService(intervalDelay);
const emitterPromiseArray = [];
// Create a TrackVADEmitter for each provided audioinput device.
for (const micDevice of micDeviceList) {
if (micDevice.kind !== 'audioinput') {
logger.warn(`Provided device ${micDevice.label} -> ${micDevice.deviceId}, is not audioinput ignoring!`);
return;
}
logger.log(`Initializing VAD context for mic: ${micDevice.label} -> ${micDevice.deviceId}`);
const rnnoiseProcessor = await createRnnoiseProcessorPromise();
const emitterPromise = JitsiMeetJS.createTrackVADEmitter(
micDevice.deviceId,
SCRIPT_NODE_SAMPLE_RATE,
rnnoiseProcessor
).then(emitter => {
emitter.on(
JitsiDetectionEvents.VAD_SCORE_PUBLISHED,
vadReportingService._devicePublishVADScore.bind(vadReportingService)
);
return {
vadEmitter: emitter,
deviceInfo: micDevice,
scoreArray: []
};
});
emitterPromiseArray.push(emitterPromise);
}
// Once all the TrackVADEmitter promises are resolved check if all of them resolved properly if not reject
// the promise and clear the already created emitters.
// $FlowFixMe - allSettled is not part of flow prototype even though it's a valid Promise function
return Promise.allSettled(emitterPromiseArray).then(outcomeArray => {
const vadContextArray = [];
const rejectedEmitterPromiseArray = [];
for (const outcome of outcomeArray) {
if (outcome.status === 'fulfilled') {
vadContextArray.push(outcome.value);
} else {
// Promise was rejected.
logger.error(`Create TrackVADEmitter promise failed with ${outcome.reason}`);
rejectedEmitterPromiseArray.push(outcome);
}
}
// Check if there were any rejected promises and clear the already created ones list.
if (rejectedEmitterPromiseArray.length > 0) {
logger.error('Cleaning up remaining VADDeviceContext, due to create fail!');
for (const context of vadContextArray) {
context.vadEmitter.destroy();
}
// Reject create promise if one emitter failed to instantiate, we might one just ignore it,
// leaving it like this for now
throw new Error('Create VADReportingService failed due to TrackVADEmitter creation issues!');
}
vadReportingService._setVADContextArray(vadContextArray);
vadReportingService._startPublish();
return vadReportingService;
});
}
/**
* Destroy TrackVADEmitters and clear the context map.
*
* @returns {void}
*/
_clearContextMap() {
for (const vadContext of this._contextMap.values()) {
vadContext.vadEmitter.destroy();
}
this._contextMap.clear();
}
/**
* Set the watched device contexts.
*
* @param {Array<VADDeviceContext>} vadContextArray - List of mics.
* @returns {void}
*/
_setVADContextArray(vadContextArray: Array<VADDeviceContext>): void {
for (const vadContext of vadContextArray) {
this._contextMap.set(vadContext.deviceInfo.deviceId, vadContext);
}
}
/**
* Start the setInterval reporting process.
*
* @returns {void}.
*/
_startPublish() {
logger.log('VADReportingService started publishing.');
this._intervalId = setInterval(() => {
this._reportVadScore();
}, this._intervalDelay);
}
/**
* Function called at set interval with selected compute. The result will be published on the set callback.
*
* @returns {void}
* @fires VAD_REPORT_PUBLISHED
*/
_reportVadScore() {
const vadComputeScoreArray = [];
const computeTimestamp = Date.now();
// Go through each device and compute cumulated VAD score.
for (const [ deviceId, vadContext ] of this._contextMap) {
const nrOfVADScores = vadContext.scoreArray.length;
let vadSum = 0;
vadContext.scoreArray.forEach(vadScore => {
vadSum += vadScore.score;
});
// TODO For now we just calculate the average score for each device, more compute algorithms will be added.
const avgVAD = vadSum / nrOfVADScores;
vadContext.scoreArray = [];
vadComputeScoreArray.push({
timestamp: computeTimestamp,
score: avgVAD,
deviceId
});
}
/**
* Once the computation for all the tracked devices is done, fire an event containing all the necessary
* information.
*
* @event VAD_REPORT_PUBLISHED
* @type Array<Object> with the following structure:
* @property {Date} timestamp - Timestamo at which the compute took place.
* @property {number} avgVAD - Average VAD score over monitored period of time.
* @property {string} deviceId - Associate local audio device ID.
*/
this.emit(VAD_REPORT_PUBLISHED, vadComputeScoreArray);
}
/**
* Callback method passed to vad emitters in order to publish their score.
*
* @param {Object} vadScore -VAD score emitted by.
* @param {Date} vadScore.timestamp - Exact time at which processed PCM sample was generated.
* @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7).
* @param {string} vadScore.deviceId - Device id of the associated track.
* @returns {void}
* @listens VAD_SCORE_PUBLISHED
*/
_devicePublishVADScore(vadScore: Object) {
const context = this._contextMap.get(vadScore.deviceId);
if (context) {
context.scoreArray.push(vadScore);
}
}
/**
* Destroy the VADReportingService, stops the setInterval reporting, destroys the emitters and clears the map.
* After this call the instance is no longer usable.
*
* @returns {void}.
*/
destroy() {
if (this._destroyed) {
return;
}
logger.log('Destroying VADReportingService.');
if (this._intervalId) {
clearInterval(this._intervalId);
this._intervalId = null;
}
this._clearContextMap();
this._destroyed = true;
}
}

4
react/features/base/tracks/actionTypes.js
View File

@ -3,11 +3,11 @@
* no data from source notification. Used to check if such a notification was previously displayed. * no data from source notification. Used to check if such a notification was previously displayed.
* *
* { * {
* type: SET_NO_SRC_DATA_NOTI_UID, * type: SET_NO_SRC_DATA_NOTIFICATION_UID,
* uid: ?number * uid: ?number
* } * }
*/ */
export const SET_NO_SRC_DATA_NOTI_UID = 'SET_NO_SRC_DATA_NOTI_UID'; export const SET_NO_SRC_DATA_NOTIFICATION_UID = 'SET_NO_SRC_DATA_NOTIFICATION_UID';
/** /**
* The type of redux action dispatched to disable screensharing or to start the * The type of redux action dispatched to disable screensharing or to start the

4
react/features/base/tracks/reducer.js
View File

@ -2,7 +2,7 @@ import { PARTICIPANT_ID_CHANGED } from '../participants';
import { ReducerRegistry, set } from '../redux'; import { ReducerRegistry, set } from '../redux';
import { import {
SET_NO_SRC_DATA_NOTI_UID, SET_NO_SRC_DATA_NOTIFICATION_UID,
TRACK_ADDED, TRACK_ADDED,
TRACK_CREATE_CANCELED, TRACK_CREATE_CANCELED,
TRACK_CREATE_ERROR, TRACK_CREATE_ERROR,
@ -140,7 +140,7 @@ ReducerRegistry.register('features/base/tracks', (state = [], action) => {
*/ */
ReducerRegistry.register('features/base/no-src-data', (state = {}, action) => { ReducerRegistry.register('features/base/no-src-data', (state = {}, action) => {
switch (action.type) { switch (action.type) {
case SET_NO_SRC_DATA_NOTI_UID: case SET_NO_SRC_DATA_NOTIFICATION_UID:
return set(state, 'noSrcDataNotificationUid', action.uid); return set(state, 'noSrcDataNotificationUid', action.uid);
default: default:

7
react/features/no-audio-signal/middleware.js
View File

@ -1,5 +1,7 @@
// @flow // @flow
import { setNoAudioSignalNotificationUid } from './actions'; import { setNoAudioSignalNotificationUid } from './actions';
import { NO_AUDIO_SIGNAL_SOUND_ID } from './constants';
import { NO_AUDIO_SIGNAL_SOUND_FILE } from './sounds';
import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app'; import { APP_WILL_MOUNT, APP_WILL_UNMOUNT } from '../base/app';
import { CONFERENCE_JOINED } from '../base/conference'; import { CONFERENCE_JOINED } from '../base/conference';
import { import {
@ -10,9 +12,7 @@ import JitsiMeetJS, { JitsiConferenceEvents } from '../base/lib-jitsi-meet';
import { MiddlewareRegistry } from '../base/redux'; import { MiddlewareRegistry } from '../base/redux';
import { updateSettings } from '../base/settings'; import { updateSettings } from '../base/settings';
import { playSound, registerSound, unregisterSound } from '../base/sounds'; import { playSound, registerSound, unregisterSound } from '../base/sounds';
import { NO_AUDIO_SIGNAL_SOUND_ID } from './constants';
import { hideNotification, showNotification } from '../notifications'; import { hideNotification, showNotification } from '../notifications';
import { NO_AUDIO_SIGNAL_SOUND_FILE } from './sounds';
MiddlewareRegistry.register(store => next => async action => { MiddlewareRegistry.register(store => next => async action => {
const result = next(action); const result = next(action);
@ -34,6 +34,7 @@ MiddlewareRegistry.register(store => next => async action => {
confAudioInputState = hasAudioInput; confAudioInputState = hasAudioInput;
// In case the notification is displayed but the conference detected audio input signal we hide it.
if (noAudioSignalNotificationUid && hasAudioInput) { if (noAudioSignalNotificationUid && hasAudioInput) {
dispatch(hideNotification(noAudioSignalNotificationUid)); dispatch(hideNotification(noAudioSignalNotificationUid));
dispatch(setNoAudioSignalNotificationUid()); dispatch(setNoAudioSignalNotificationUid());
@ -50,7 +51,7 @@ MiddlewareRegistry.register(store => next => async action => {
} }
// Force the flag to false in case AUDIO_INPUT_STATE_CHANGE is received after the notification is displayed, // Force the flag to false in case AUDIO_INPUT_STATE_CHANGE is received after the notification is displayed,
// thus making sure we check properly if the notification should display. // possibly preventing the notification from displaying because of an outdated state.
confAudioInputState = false; confAudioInputState = false;