jiti-meet/react/features/facial-recognition/actions.js

248 lines
7.1 KiB
JavaScript
Raw Normal View History

feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
// @flow
import { getLocalVideoTrack } from '../base/tracks';
import 'image-capture';
import './createImageBitmap';
import {
ADD_FACIAL_EXPRESSION,
SET_DETECTION_TIME_INTERVAL,
START_FACIAL_RECOGNITION,
STOP_FACIAL_RECOGNITION
} from './actionTypes';
import { sendDataToWorker } from './functions';
import logger from './logger';
/**
* Time used for detection interval when facial expressions worker uses webgl backend.
*/
const WEBGL_TIME_INTERVAL = 1000;
/**
* Time used for detection interval when facial expression worker uses cpu backend.
*/
const CPU_TIME_INTERVAL = 6000;
/**
* Object containing a image capture of the local track.
*/
let imageCapture;
/**
* Object where the facial expression worker is stored.
*/
let worker;
/**
* The last facial expression received from the worker.
*/
let lastFacialExpression;
/**
* How many duplicate consecutive expression occurred.
* If a expression that is not the same as the last one it is reset to 0.
*/
let duplicateConsecutiveExpressions = 0;
/**
* Loads the worker that predicts the facial expression.
*
* @returns {void}
*/
export function loadWorker() {
return function(dispatch: Function) {
if (!window.Worker) {
logger.warn('Browser does not support web workers');
return;
}
let baseUrl = '';
const app: Object = document.querySelector('script[src*="app.bundle.min.js"]');
if (app) {
const idx = app.src.lastIndexOf('/');
baseUrl = `${app.src.substring(0, idx)}/`;
}
let workerUrl = `${baseUrl}facial-expressions-worker.min.js`;
const workerBlob = new Blob([ `importScripts("${workerUrl}");` ], { type: 'application/javascript' });
workerUrl = window.URL.createObjectURL(workerBlob);
worker = new Worker(workerUrl, { name: 'Facial Expression Worker' });
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
worker.onmessage = function(e: Object) {
const { type, value } = e.data;
// receives a message indicating what type of backend tfjs decided to use.
// it is received after as a response to the first message sent to the worker.
if (type === 'tf-backend' && value) {
let detectionTimeInterval = -1;
if (value === 'webgl') {
detectionTimeInterval = WEBGL_TIME_INTERVAL;
} else if (value === 'cpu') {
detectionTimeInterval = CPU_TIME_INTERVAL;
}
dispatch(setDetectionTimeInterval(detectionTimeInterval));
}
// receives a message with the predicted facial expression.
if (type === 'facial-expression') {
sendDataToWorker(worker, imageCapture);
if (!value) {
return;
}
if (value === lastFacialExpression) {
duplicateConsecutiveExpressions++;
} else {
lastFacialExpression
&& dispatch(addFacialExpression(lastFacialExpression, duplicateConsecutiveExpressions + 1));
lastFacialExpression = value;
duplicateConsecutiveExpressions = 0;
}
}
};
worker.postMessage({
id: 'SET_MODELS_URL',
url: baseUrl
});
dispatch(startFacialRecognition());
feat(facial-expressions): add the facial expression feature and display them in speakerstats (#10006) * Initial implementation; Happy flow * Maybe revert this * Functional prototype * feat(facial-expressions): get stream when changing background effect and use presenter effect with camera * add(facial-expressions): array that stores the expressions durin the meeting * refactor(facial-expressions): capture imagebitmap from stream with imagecapture api * add(speaker-stats): expression label * fix(facial-expression): expression store * revert: expression leabel on speaker stats * add(facial-expressions): broadcast of expression when it changes * feat: facial expression handling on prosody * fix(facial-expressions): get the right track when opening and closing camera * add(speaker-stats): facial expression column * fix(facial-expressions): allow to start facial recognition only after joining conference * fix(mod_speakerstats_component): storing last emotion in speaker stats component and sending it * chore(facial-expressions): change detection from 2000ms to 1000ms * add(facial-expressions): send expression to server when there is only one participant * feat(facial-expressions): store expresions as a timeline * feat(mod_speakerstats_component): store facial expresions as a timeline * fix(facial-expressions): stop facial recognition only when muting video track * fix(facial-expressions): presenter mode get right track to detect face * add: polyfils for image capture for firefox and safari * refactor(facial-expressions): store expressions by counting them in a map * chore(facial-expressions): remove manually assigning the backend for tenserflowjs * feat(facial-expressions): move face-api from main thread to web worker * fix(facial-expressions): make feature work on firefox and safari * feat(facial-expressions): camera time tracker * feat(facial-expressions): camera time tracker in prosody * add(facial-expressions): expressions time as TimeElapsed object in speaker stats * fix(facial-expresions): lower the frequency of detection when tf uses cpu backend * add(facial-expressions): duration to the expression and send it with durantion when it is done * fix(facial-expressions): prosody speaker stats covert fro string to number and bool values set by xmpp * refactor(facial-expressions): change expressions labels from text to emoji * refactor(facial-expressions): remove camera time tracker * add(facial-expressions): detection time interval * chore(facial-expressions): add docs and minor refactor of the code * refactor(facial-expressions): put timeout in worker and remove set interval in main thread * feat(facial-expressions): disable feature in the config * add(facial-expressions): tooltips of labels in speaker stats * refactor(facial-expressions): send facial expressions function and remove some unused functions and console logs * refactor(facial-expressions): rename action type when a change is done to the track by the virtual backgrounds to be used in facial expressions middleware * chore(facial-expressions): order imports and format some code * fix(facial-expressions): rebase issues with newer master * fix(facial-expressions): package-lock.json * fix(facial-expression): add commented default value of disableFacialRecognition flag and short description * fix(facial-expressions): change disableFacialRecognition to enableFacialRecognition flag in config * fix: resources load-test package-lock.json * fix(facial-expressions): set and get facial expressions only if facial recognition enabled * add: facial recognition resources folder in .eslintignore * chore: package-lock update * fix: package-lock.json * fix(facial-expressions): gpu memory leak in the web worker * fix(facial-expressions): set cpu time interval for detection to 6000ms * chore(speaker-stats): fix indentation * chore(facial-expressions): remove empty lines between comments and type declarations * fix(facial-expressions): remove camera timetracker * fix(facial-expressions): remove facialRecognitionAllowed flag * fix(facial-expressions): remove sending interval time to worker * refactor(facial-expression): middleware * fix(facial-expression): end tensor scope after setting backend * fix(facial-expressions): sending info back to worker only on facial expression message * fix: lint errors * refactor(facial-expressions): bundle web worker using webpack * fix: deploy-facial-expressions command in makefile * chore: fix load test package-lock.json and package.json * chore: sync package-lock.json Co-authored-by: Mihai-Andrei Uscat <mihai.uscat@8x8.com>
2021-11-17 14:33:03 +00:00
};
}
/**
* Starts the recognition and detection of face expressions.
*
* @param {Object} stream - Video stream.
* @returns {Function}
*/
export function startFacialRecognition() {
return async function(dispatch: Function, getState: Function) {
if (worker === undefined || worker === null) {
return;
}
const state = getState();
const { recognitionActive } = state['features/facial-recognition'];
if (recognitionActive) {
return;
}
const localVideoTrack = getLocalVideoTrack(state['features/base/tracks']);
if (localVideoTrack === undefined) {
return;
}
const stream = localVideoTrack.jitsiTrack.getOriginalStream();
if (stream === null) {
return;
}
dispatch({ type: START_FACIAL_RECOGNITION });
logger.log('Start face recognition');
const firstVideoTrack = stream.getVideoTracks()[0];
// $FlowFixMe
imageCapture = new ImageCapture(firstVideoTrack);
sendDataToWorker(worker, imageCapture);
};
}
/**
* Stops the recognition and detection of face expressions.
*
* @returns {void}
*/
export function stopFacialRecognition() {
return function(dispatch: Function, getState: Function) {
const state = getState();
const { recognitionActive } = state['features/facial-recognition'];
if (!recognitionActive) {
imageCapture = null;
return;
}
imageCapture = null;
worker.postMessage({
id: 'CLEAR_TIMEOUT'
});
lastFacialExpression
&& dispatch(addFacialExpression(lastFacialExpression, duplicateConsecutiveExpressions + 1));
duplicateConsecutiveExpressions = 0;
dispatch({ type: STOP_FACIAL_RECOGNITION });
logger.log('Stop face recognition');
};
}
/**
* Resets the track in the image capture.
*
* @returns {void}
*/
export function resetTrack() {
return function(dispatch: Function, getState: Function) {
const state = getState();
const { jitsiTrack: localVideoTrack } = getLocalVideoTrack(state['features/base/tracks']);
const stream = localVideoTrack.getOriginalStream();
const firstVideoTrack = stream.getVideoTracks()[0];
// $FlowFixMe
imageCapture = new ImageCapture(firstVideoTrack);
};
}
/**
* Changes the track from the image capture with a given one.
*
* @param {Object} track - The track that will be in the new image capture.
* @returns {void}
*/
export function changeTrack(track: Object) {
const { jitsiTrack } = track;
const stream = jitsiTrack.getOriginalStream();
const firstVideoTrack = stream.getVideoTracks()[0];
// $FlowFixMe
imageCapture = new ImageCapture(firstVideoTrack);
}
/**
* Adds a new facial expression and its duration.
*
* @param {string} facialExpression - Facial expression to be added.
* @param {number} duration - Duration in seconds of the facial expression.
* @returns {Object}
*/
function addFacialExpression(facialExpression: string, duration: number) {
return function(dispatch: Function, getState: Function) {
const { detectionTimeInterval } = getState()['features/facial-recognition'];
let finalDuration = duration;
if (detectionTimeInterval !== -1) {
finalDuration *= detectionTimeInterval / 1000;
}
dispatch({
type: ADD_FACIAL_EXPRESSION,
facialExpression,
duration: finalDuration
});
};
}
/**
* Sets the time interval for the detection worker post message.
*
* @param {number} time - The time interval.
* @returns {Object}
*/
function setDetectionTimeInterval(time: number) {
return {
type: SET_DETECTION_TIME_INTERVAL,
time
};
}