jiti-meet/react/features/vad-reporter/VADReportingService.js

285 lines
9.2 KiB
JavaScript
Raw Normal View History

// @flow
import EventEmitter from 'events';
import logger from './logger';
import TrackVADEmitter from './TrackVADEmitter';
import { VAD_SCORE_PUBLISHED, VAD_REPORT_PUBLISHED } from './VADEvents';
import type { VADScore } from './TrackVADEmitter';
export type { VADScore };
/**
* Sample rate used by TrackVADEmitter, this value determines how often the ScriptProcessorNode is going to call the
* process audio function and with what sample size.
* Basically lower values mean more callbacks with lower processing times bigger values less callbacks with longer
* processing times. This value is somewhere in the middle, so we strike a balance between flooding with callbacks
* and processing time. Possible values 256, 512, 1024, 2048, 4096, 8192, 16384. Passing other values will default
* to closes neighbor.
*/
const SCRIPT_NODE_SAMPLE_RATE = 4096;
/**
* Context that contains the emitter and additional information about the device.
*/
type VADDeviceContext = {
/**
* MediaDeviceInfo for associated context
*/
deviceInfo: MediaDeviceInfo,
/**
* Array with VAD scores publish from the emitter.
*/
scoreArray: Array<VADScore>,
/**
* TrackVADEmitter associated with media device
*/
vadEmitter: TrackVADEmitter
};
/**
* The structure used by VADReportingService to relay a score report
*/
export type VADReportScore = {
/**
* Device ID associated with the VAD score
*/
deviceId: string,
/**
* The PCM score from 0 - 1 i.e. 0.60
*/
score: number,
/**
* Epoch time at which PCM was recorded
*/
timestamp: number
};
/**
* Voice activity detection reporting service. The service create TrackVADEmitters for the provided devices and
* publishes an average of their VAD score over the specified interval via EventEmitter.
* The service is not reusable if destroyed a new one needs to be created, i.e. when a new device is added to the system
* a new service needs to be created and the old discarded.
*/
export default class VADReportingService extends EventEmitter {
/**
* Map containing context for devices currently being monitored by the reporting service.
*/
_contextMap: Map<string, VADDeviceContext>;
/**
* State flag, check if the instance was destroyed.
*/
_destroyed: boolean = false;
/**
* Delay at which to publish VAD score for monitored devices.
*/
_intervalDelay: number;
/**
* Identifier for the interval publishing stats on the set interval.
*/
_intervalId: ?IntervalID;
/**
* Constructor.
*
* @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices.
* @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score.
*/
constructor(intervalDelay: number) {
super();
this._contextMap = new Map();
this._intervalDelay = intervalDelay;
logger.log(`Constructed VADReportingService with publish interval of: ${intervalDelay}`);
}
/**
* Factory methods that creates the TrackVADEmitters for the associated array of devices and instantiates
* a VADReportingService.
*
* @param {Array<MediaDeviceInfo>} micDeviceList - Device list that is monitored inside the service.
* @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices.
* @param {Function} publishScoreCallBack - Function called on the specific interval with the calculated VAD score.
*
* @returns {Promise<VADReportingService>}
*/
static create(micDeviceList: Array<MediaDeviceInfo>, intervalDelay: number) {
const vadReportingService = new VADReportingService(intervalDelay);
const emitterPromiseArray = [];
// Create a TrackVADEmitter for each provided audioinput device.
for (const micDevice of micDeviceList) {
if (micDevice.kind !== 'audioinput') {
logger.warn(`Provided device ${micDevice.label} -> ${micDevice.deviceId}, is not audioinput ignoring!`);
return;
}
logger.log(`Initializing VAD context for mic: ${micDevice.label} -> ${micDevice.deviceId}`);
const emitterPromise = TrackVADEmitter.create(micDevice.deviceId, SCRIPT_NODE_SAMPLE_RATE).then(emitter => {
emitter.on(VAD_SCORE_PUBLISHED, vadReportingService._devicePublishVADScore.bind(vadReportingService));
return {
vadEmitter: emitter,
deviceInfo: micDevice,
scoreArray: []
};
});
emitterPromiseArray.push(emitterPromise);
}
// Once all the TrackVADEmitter promises are resolved check if all of them resolved properly if not reject
// the promise and clear the already created emitters.
// $FlowFixMe - allSettled is not part of flow prototype even though it's a valid Promise function
return Promise.allSettled(emitterPromiseArray).then(outcomeArray => {
const vadContextArray = [];
const rejectedEmitterPromiseArray = [];
for (const outcome of outcomeArray) {
if (outcome.status === 'fulfilled') {
vadContextArray.push(outcome.value);
} else {
// Promise was rejected.
logger.error(`Create TrackVADEmitter promise failed with ${outcome.reason}`);
rejectedEmitterPromiseArray.push(outcome);
}
}
// Check if there were any rejected promises and clear the already created ones list.
if (rejectedEmitterPromiseArray.length > 0) {
logger.error('Cleaning up remaining VADDeviceContext, due to create fail!');
for (const context of vadContextArray) {
context.vadEmitter.destroy();
}
// Reject create promise if one emitter failed to instantiate, we might one just ignore it,
// leaving it like this for now
throw new Error('Create VADReportingService failed due to TrackVADEmitter creation issues!');
}
vadReportingService._setVADContextArray(vadContextArray);
vadReportingService._startPublish();
return vadReportingService;
});
}
/**
* Destroy TrackVADEmitters and clear the context map.
*
* @returns {void}
*/
_clearContextMap() {
for (const vadContext of this._contextMap.values()) {
vadContext.vadEmitter.destroy();
}
this._contextMap.clear();
}
/**
* Set the watched device contexts.
*
* @param {Array<VADDeviceContext>} vadContextArray - List of mics.
* @returns {void}
*/
_setVADContextArray(vadContextArray: Array<VADDeviceContext>): void {
for (const vadContext of vadContextArray) {
this._contextMap.set(vadContext.deviceInfo.deviceId, vadContext);
}
}
/**
* Start the setInterval reporting process.
*
* @returns {void}.
*/
_startPublish() {
logger.log('VADReportingService started publishing.');
this._intervalId = setInterval(() => {
this._reportVadScore();
}, this._intervalDelay);
}
/**
* Function called at set interval with selected compute. The result will be published on the set callback.
*
* @returns {void}
*/
_reportVadScore() {
const vadComputeScoreArray = [];
const computeTimestamp = Date.now();
// Go through each device and compute cumulated VAD score.
for (const [ deviceId, vadContext ] of this._contextMap) {
const nrOfVADScores = vadContext.scoreArray.length;
let vadSum = 0;
vadContext.scoreArray.forEach(vadScore => {
vadSum += vadScore.score;
});
// TODO For now we just calculate the average score for each device, more compute algorithms will be added.
const avgVAD = vadSum / nrOfVADScores;
vadContext.scoreArray = [];
vadComputeScoreArray.push({
timestamp: computeTimestamp,
score: avgVAD,
deviceId
});
}
this.emit(VAD_REPORT_PUBLISHED, vadComputeScoreArray);
}
/**
* Callback method passed to vad emitters in order to publish their score.
*
* @param {VADScore} vadScore - Mic publishing the score.
* @returns {void}
*/
_devicePublishVADScore(vadScore: VADScore) {
const context = this._contextMap.get(vadScore.deviceId);
if (context) {
context.scoreArray.push(vadScore);
}
}
/**
* Destroy the VADReportingService, stops the setInterval reporting, destroys the emitters and clears the map.
* After this call the instance is no longer usable.
*
* @returns {void}.
*/
destroy() {
if (this._destroyed) {
return;
}
logger.log('Destroying VADReportingService.');
if (this._intervalId) {
clearInterval(this._intervalId);
this._intervalId = null;
}
this._clearContextMap();
this._destroyed = true;
}
}