jiti-meet/react/features/face-landmarks/actions.js

// @flow
import 'image-capture';
import './createImageBitmap';

import { getCurrentConference } from '../base/conference';
import { getLocalParticipant, getParticipantCount } from '../base/participants';
import { getLocalVideoTrack } from '../base/tracks';
import { getBaseUrl } from '../base/util';

import {
    ADD_FACE_EXPRESSION,
    ADD_TO_FACE_EXPRESSIONS_BUFFER,
    CLEAR_FACE_EXPRESSIONS_BUFFER,
    START_FACE_LANDMARKS_DETECTION,
    STOP_FACE_LANDMARKS_DETECTION,
    UPDATE_FACE_COORDINATES
} from './actionTypes';
import {
    DETECTION_TYPES,
    INIT_WORKER,
    WEBHOOK_SEND_TIME_INTERVAL
} from './constants';
import {
    getDetectionInterval,
    sendDataToWorker,
    sendFaceBoxToParticipants,
    sendFaceExpressionsWebhook
} from './functions';
import logger from './logger';

declare var APP: Object;

/**
 * Object containing  a image capture of the local track.
 */
let imageCapture;

/**
 * Object where the face landmarks worker is stored.
 */
let worker;

/**
 * The last face expression received from the worker.
 */
let lastFaceExpression;

/**
 * The last face expression timestamp.
 */
let lastFaceExpressionTimestamp;

/**
 * How many duplicate consecutive expression occurred.
 * If a expression that is not the same as the last one it is reset to 0.
 */
let duplicateConsecutiveExpressions = 0;

/**
 * Variable that keeps the interval for sending expressions to webhook.
 */
let webhookSendInterval;

/**
 * Variable that keeps the interval for detecting faces in a frame.
 */
let detectionInterval;

/**
 * Loads the worker that detects the face landmarks.
 *
 * @returns {void}
 */
export function loadWorker() {
    return function(dispatch: Function, getState: Function) {
        if (worker) {
            logger.info('Worker has already been initialized');

            return;
        }

        if (navigator.product === 'ReactNative') {
            logger.warn('Unsupported environment for face recognition');

            return;
        }

        const baseUrl = `${getBaseUrl()}libs/`;
        let workerUrl = `${baseUrl}face-landmarks-worker.min.js`;

        const workerBlob = new Blob([ `importScripts("${workerUrl}");` ], { type: 'application/javascript' });

        workerUrl = window.URL.createObjectURL(workerBlob);
        worker = new Worker(workerUrl, { name: 'Face Recognition Worker' });
        worker.onmessage = function(e: Object) {
            const { faceExpression, faceBox } = e.data;

            if (faceExpression) {
                if (faceExpression === lastFaceExpression) {
                    duplicateConsecutiveExpressions++;
                } else {
                    if (lastFaceExpression && lastFaceExpressionTimestamp) {
                        dispatch(addFaceExpression(
                            lastFaceExpression,
                            duplicateConsecutiveExpressions + 1,
                            lastFaceExpressionTimestamp
                        ));
                    }
                    lastFaceExpression = faceExpression;
                    lastFaceExpressionTimestamp = Date.now();
                    duplicateConsecutiveExpressions = 0;
                }
            }

            if (faceBox) {
                const state = getState();
                const conference = getCurrentConference(state);
                const localParticipant = getLocalParticipant(state);

                if (getParticipantCount(state) > 1) {
                    sendFaceBoxToParticipants(conference, faceBox);
                }

                dispatch({
                    type: UPDATE_FACE_COORDINATES,
                    faceBox,
                    id: localParticipant.id
                });
            }

            APP.API.notifyFaceLandmarkDetected(faceBox, faceExpression);
        };

        const { faceLandmarks } = getState()['features/base/config'];
        const detectionTypes = [
            faceLandmarks?.enableFaceCentering && DETECTION_TYPES.FACE_BOX,
            faceLandmarks?.enableFaceExpressionsDetection && DETECTION_TYPES.FACE_EXPRESSIONS
        ].filter(Boolean);

        worker.postMessage({
            type: INIT_WORKER,
            baseUrl,
            detectionTypes
        });

        dispatch(startFaceLandmarksDetection());
    };
}

/**
 * Starts the recognition and detection of face expressions.
 *
 * @param {Track | undefined} track - Track for which to start detecting faces.
 * @returns {Function}
 */
export function startFaceLandmarksDetection(track) {
    return async function(dispatch: Function, getState: Function) {
        if (!worker) {
            return;
        }

        const state = getState();
        const { recognitionActive } = state['features/face-landmarks'];

        if (recognitionActive) {
            logger.log('Face recognition already active.');

            return;
        }

        const localVideoTrack = track || getLocalVideoTrack(state['features/base/tracks']);

        if (localVideoTrack === undefined) {
            logger.warn('Face landmarks detection is disabled due to missing local track.');

            return;
        }

        const stream = localVideoTrack.jitsiTrack.getOriginalStream();

        dispatch({ type: START_FACE_LANDMARKS_DETECTION });
        logger.log('Start face recognition');

        const firstVideoTrack = stream.getVideoTracks()[0];
        const { faceLandmarks } = state['features/base/config'];

        imageCapture = new ImageCapture(firstVideoTrack);

        detectionInterval = setInterval(() => {
            sendDataToWorker(
                worker,
                imageCapture,
                faceLandmarks?.faceCenteringThreshold
            );
        }, getDetectionInterval(state));

        if (faceLandmarks?.enableFaceExpressionsDetection) {
            webhookSendInterval = setInterval(async () => {
                const result = await sendFaceExpressionsWebhook(getState());

                if (result) {
                    dispatch(clearFaceExpressionBuffer());
                }
            }, WEBHOOK_SEND_TIME_INTERVAL);
        }
    };
}

/**
 * Stops the recognition and detection of face expressions.
 *
 * @returns {void}
 */
export function stopFaceLandmarksDetection() {
    return function(dispatch: Function, getState: Function) {
        const { recognitionActive } = getState()['features/face-landmarks'];

        if (lastFaceExpression && lastFaceExpressionTimestamp && recognitionActive) {
            dispatch(
                addFaceExpression(
                    lastFaceExpression,
                    duplicateConsecutiveExpressions + 1,
                    lastFaceExpressionTimestamp
                )
            );
        }

        clearInterval(webhookSendInterval);
        clearInterval(detectionInterval);

        duplicateConsecutiveExpressions = 0;
        lastFaceExpression = null;
        lastFaceExpressionTimestamp = null;
        webhookSendInterval = null;
        detectionInterval = null;
        imageCapture = null;

        dispatch({ type: STOP_FACE_LANDMARKS_DETECTION });
        logger.log('Stop face recognition');
    };
}

/**
 * Adds a new face expression and its duration.
 *
 * @param  {string} faceExpression - Face expression to be added.
 * @param  {number} duration - Duration in seconds of the face expression.
 * @param  {number} timestamp - Duration in seconds of the face expression.
 * @returns {Object}
 */
function addFaceExpression(faceExpression: string, duration: number, timestamp: number) {
    return function(dispatch: Function, getState: Function) {
        const finalDuration = duration * getDetectionInterval(getState()) / 1000;

        dispatch({
            type: ADD_FACE_EXPRESSION,
            faceExpression,
            duration: finalDuration,
            timestamp
        });
    };
}

/**
 * Adds a face expression with its timestamp to the face expression buffer.
 *
 * @param  {Object} faceExpression - Object containing face expression string and its timestamp.
 * @returns {Object}
 */
export function addToFaceExpressionsBuffer(faceExpression: Object) {
    return {
        type: ADD_TO_FACE_EXPRESSIONS_BUFFER,
        faceExpression
    };
}

/**
 * Clears the face expressions array in the state.
 *
 * @returns {Object}
 */
function clearFaceExpressionBuffer() {
    return {
        type: CLEAR_FACE_EXPRESSIONS_BUFFER
    };
}