jiti-meet/react/features/facial-recognition/actions.js

302 lines
8.6 KiB
JavaScript

// @flow
import 'image-capture';
import './createImageBitmap';
import { getLocalVideoTrack } from '../base/tracks';
import { getBaseUrl } from '../base/util';
import {
ADD_FACIAL_EXPRESSION,
ADD_TO_FACIAL_EXPRESSIONS_BUFFER,
CLEAR_FACIAL_EXPRESSIONS_BUFFER,
SET_DETECTION_TIME_INTERVAL,
START_FACIAL_RECOGNITION,
STOP_FACIAL_RECOGNITION
} from './actionTypes';
import {
CLEAR_TIMEOUT,
FACIAL_EXPRESSION_MESSAGE,
INIT_WORKER,
INTERVAL_MESSAGE,
WEBHOOK_SEND_TIME_INTERVAL
} from './constants';
import { sendDataToWorker, sendFacialExpressionsWebhook } from './functions';
import logger from './logger';
/**
* Object containing a image capture of the local track.
*/
let imageCapture;
/**
* Object where the facial expression worker is stored.
*/
let worker;
/**
* The last facial expression received from the worker.
*/
let lastFacialExpression;
/**
* The last facial expression timestamp.
*/
let lastFacialExpressionTimestamp;
/**
* How many duplicate consecutive expression occurred.
* If a expression that is not the same as the last one it is reset to 0.
*/
let duplicateConsecutiveExpressions = 0;
/**
* Variable that keeps the interval for sending expressions to webhook.
*/
let sendInterval;
/**
* Loads the worker that predicts the facial expression.
*
* @returns {void}
*/
export function loadWorker() {
return function(dispatch: Function) {
if (!window.Worker) {
logger.warn('Browser does not support web workers');
return;
}
const baseUrl = `${getBaseUrl()}/libs/`;
let workerUrl = `${baseUrl}facial-expressions-worker.min.js`;
const workerBlob = new Blob([ `importScripts("${workerUrl}");` ], { type: 'application/javascript' });
workerUrl = window.URL.createObjectURL(workerBlob);
worker = new Worker(workerUrl, { name: 'Facial Expression Worker' });
worker.onmessage = function(e: Object) {
const { type, value } = e.data;
// receives a message indicating what type of backend tfjs decided to use.
// it is received after as a response to the first message sent to the worker.
if (type === INTERVAL_MESSAGE) {
value && dispatch(setDetectionTimeInterval(value));
}
// receives a message with the predicted facial expression.
if (type === FACIAL_EXPRESSION_MESSAGE) {
sendDataToWorker(worker, imageCapture);
if (!value) {
return;
}
if (value === lastFacialExpression) {
duplicateConsecutiveExpressions++;
} else {
if (lastFacialExpression && lastFacialExpressionTimestamp) {
dispatch(
addFacialExpression(
lastFacialExpression,
duplicateConsecutiveExpressions + 1,
lastFacialExpressionTimestamp
)
);
}
lastFacialExpression = value;
lastFacialExpressionTimestamp = Date.now();
duplicateConsecutiveExpressions = 0;
}
}
};
worker.postMessage({
type: INIT_WORKER,
url: baseUrl,
windowScreenSize: window.screen ? {
width: window.screen.width,
height: window.screen.height
} : undefined
});
dispatch(startFacialRecognition());
};
}
/**
* Starts the recognition and detection of face expressions.
*
* @param {Object} stream - Video stream.
* @returns {Function}
*/
export function startFacialRecognition() {
return async function(dispatch: Function, getState: Function) {
if (!worker) {
return;
}
const state = getState();
const { recognitionActive } = state['features/facial-recognition'];
if (recognitionActive) {
return;
}
const localVideoTrack = getLocalVideoTrack(state['features/base/tracks']);
if (localVideoTrack === undefined) {
return;
}
const stream = localVideoTrack.jitsiTrack.getOriginalStream();
if (stream === null) {
return;
}
dispatch({ type: START_FACIAL_RECOGNITION });
logger.log('Start face recognition');
const firstVideoTrack = stream.getVideoTracks()[0];
// $FlowFixMe
imageCapture = new ImageCapture(firstVideoTrack);
sendDataToWorker(worker, imageCapture);
sendInterval = setInterval(async () => {
const result = await sendFacialExpressionsWebhook(getState());
if (result) {
dispatch(clearFacialExpressionBuffer());
}
}
, WEBHOOK_SEND_TIME_INTERVAL);
};
}
/**
* Stops the recognition and detection of face expressions.
*
* @returns {void}
*/
export function stopFacialRecognition() {
return function(dispatch: Function, getState: Function) {
const state = getState();
const { recognitionActive } = state['features/facial-recognition'];
if (!recognitionActive) {
imageCapture = null;
return;
}
imageCapture = null;
worker.postMessage({
type: CLEAR_TIMEOUT
});
if (lastFacialExpression && lastFacialExpressionTimestamp) {
dispatch(
addFacialExpression(
lastFacialExpression,
duplicateConsecutiveExpressions + 1,
lastFacialExpressionTimestamp
)
);
}
duplicateConsecutiveExpressions = 0;
if (sendInterval) {
clearInterval(sendInterval);
sendInterval = null;
}
dispatch({ type: STOP_FACIAL_RECOGNITION });
logger.log('Stop face recognition');
};
}
/**
* Resets the track in the image capture.
*
* @returns {void}
*/
export function resetTrack() {
return function(dispatch: Function, getState: Function) {
const state = getState();
const { jitsiTrack: localVideoTrack } = getLocalVideoTrack(state['features/base/tracks']);
const stream = localVideoTrack.getOriginalStream();
const firstVideoTrack = stream.getVideoTracks()[0];
// $FlowFixMe
imageCapture = new ImageCapture(firstVideoTrack);
};
}
/**
* Changes the track from the image capture with a given one.
*
* @param {Object} track - The track that will be in the new image capture.
* @returns {void}
*/
export function changeTrack(track: Object) {
const { jitsiTrack } = track;
const stream = jitsiTrack.getOriginalStream();
const firstVideoTrack = stream.getVideoTracks()[0];
// $FlowFixMe
imageCapture = new ImageCapture(firstVideoTrack);
}
/**
* Adds a new facial expression and its duration.
*
* @param {string} facialExpression - Facial expression to be added.
* @param {number} duration - Duration in seconds of the facial expression.
* @param {number} timestamp - Duration in seconds of the facial expression.
* @returns {Object}
*/
function addFacialExpression(facialExpression: string, duration: number, timestamp: number) {
return function(dispatch: Function, getState: Function) {
const { detectionTimeInterval } = getState()['features/facial-recognition'];
let finalDuration = duration;
if (detectionTimeInterval !== -1) {
finalDuration *= detectionTimeInterval / 1000;
}
dispatch({
type: ADD_FACIAL_EXPRESSION,
facialExpression,
duration: finalDuration,
timestamp
});
};
}
/**
* Sets the time interval for the detection worker post message.
*
* @param {number} time - The time interval.
* @returns {Object}
*/
function setDetectionTimeInterval(time: number) {
return {
type: SET_DETECTION_TIME_INTERVAL,
time
};
}
/**
* Adds a facial expression with its timestamp to the facial expression buffer.
*
* @param {Object} facialExpression - Object containing facial expression string and its timestamp.
* @returns {Object}
*/
export function addToFacialExpressionsBuffer(facialExpression: Object) {
return {
type: ADD_TO_FACIAL_EXPRESSIONS_BUFFER,
facialExpression
};
}
/**
* Clears the facial expressions array in the state.
*
* @returns {Object}
*/
function clearFacialExpressionBuffer() {
return {
type: CLEAR_FACIAL_EXPRESSIONS_BUFFER
};
}