feat(rtc-stats): send face landmarks detection off timestamp to service (#12183)

* feat(rtc-stats): send camera off timestamp to service

* code review

* improve error handling

* improve rtcstats middleware and complete typescript types

react/features/face-landmarks/FaceLandmarksDetector.ts

@@ -6,13 +6,18 @@ import { IStore } from '../app/types';
 import { getLocalVideoTrack } from '../base/tracks/functions';
 import { getBaseUrl } from '../base/util/helpers';
 
-import { NEW_FACE_COORDINATES } from './actionTypes';
-import { addFaceExpression, clearFaceExpressionBuffer } from './actions';
+import {
+    addFaceExpression,
+    faceLandmarkDetectionStopped,
+    clearFaceExpressionBuffer,
+    newFaceBox
+} from './actions';
 import {
     DETECTION_TYPES,
     INIT_WORKER,
     DETECT_FACE,
-    WEBHOOK_SEND_TIME_INTERVAL
+    WEBHOOK_SEND_TIME_INTERVAL,
+    FACE_LANDMARK_DETECTION_ERROR_THRESHOLD
 } from './constants';
 import {
     getDetectionInterval,
@@ -38,6 +43,7 @@ class FaceLandmarksDetector {
     private recognitionActive = false;
     private canvas?: HTMLCanvasElement;
     private context?: CanvasRenderingContext2D | null;
+    private errorCount = 0;
 
     /**
      * Constructor for class, checks if the environment supports OffscreenCanvas.
@@ -119,10 +125,7 @@ class FaceLandmarksDetector {
             }
 
             if (faceBox) {
-                dispatch({
-                    type: NEW_FACE_COORDINATES,
-                    faceBox
-                });
+                dispatch(newFaceBox(faceBox));
             }
 
             APP.API.notifyFaceLandmarkDetected(faceBox, faceExpression);
@@ -187,10 +190,15 @@ class FaceLandmarksDetector {
 
             if (this.worker && this.imageCapture) {
                 this.sendDataToWorker(
-                    this.imageCapture,
                     faceLandmarks?.faceCenteringThreshold
                 ).then(status => {
-                    if (!status) {
+                    if (status) {
+                        this.errorCount = 0;
+                    } else if (++this.errorCount > FACE_LANDMARK_DETECTION_ERROR_THRESHOLD) {
+                        /* this prevents the detection from stopping immediately after occurring an error
+                         * sometimes due to the small detection interval when starting the detection some errors
+                         * might occur due to the track not being ready
+                        */
                         this.stopDetection({
                             dispatch,
                             getState
@@ -243,19 +251,22 @@ class FaceLandmarksDetector {
         this.detectionInterval = null;
         this.imageCapture = null;
         this.recognitionActive = false;
+        dispatch(faceLandmarkDetectionStopped(Date.now()));
         logger.log('Stop face detection');
     }
 
     /**
      * Sends the image data a canvas from the track in the image capture to the face detection worker.
      *
-     * @param {Object} imageCapture - Image capture that contains the current track.
      * @param {number} faceCenteringThreshold  - Movement threshold as percentage for sharing face coordinates.
      * @returns {Promise<boolean>} - True if sent, false otherwise.
      */
-    private async sendDataToWorker(imageCapture: ImageCapture, faceCenteringThreshold = 10): Promise<boolean> {
-        if (!imageCapture || !this.worker) {
-            logger.log('Could not send data to worker');
+    private async sendDataToWorker(faceCenteringThreshold = 10): Promise<boolean> {
+        if (!this.imageCapture
+            || !this.worker
+            || !this.imageCapture?.track
+            || this.imageCapture?.track.readyState !== 'live') {
+            logger.log('Environment not ready! Could not send data to worker');
 
             return false;
         }
@@ -264,10 +275,9 @@ class FaceLandmarksDetector {
         let image;
 
         try {
-            imageBitmap = await imageCapture.grabFrame();
+            imageBitmap = await this.imageCapture.grabFrame();
         } catch (err) {
             logger.log('Could not send data to worker');
-            logger.warn(err);
 
             return false;
         }

react/features/face-landmarks/actionTypes.ts

@@ -49,3 +49,12 @@ export const UPDATE_FACE_COORDINATES = 'UPDATE_FACE_COORDINATES';
  * }
  */
  export const NEW_FACE_COORDINATES = 'NEW_FACE_COORDINATES';
+
+ /**
+  * Redux action type dispatched in order to signal that the face landmarks detection stopped.
+  * {
+  *      type: FACE_LANDMARK_DETECTION_STOPPED,
+  *      timestamp: number,
+  * }
+*/
+export const FACE_LANDMARK_DETECTION_STOPPED = 'FACE_LANDMARK_DETECTION_STOPPED';

react/features/face-landmarks/actions.ts

@@ -5,6 +5,7 @@ import { AnyAction } from 'redux';
 import {
     ADD_FACE_EXPRESSION,
     ADD_TO_FACE_EXPRESSIONS_BUFFER,
+    FACE_LANDMARK_DETECTION_STOPPED,
     CLEAR_FACE_EXPRESSIONS_BUFFER,
     NEW_FACE_COORDINATES
 } from './actionTypes';
@@ -68,3 +69,16 @@ export function newFaceBox(faceBox: FaceBox): AnyAction {
         faceBox
     };
 }
+
+/**
+ * Dispatches the face landmarks detection stopped event in order to be sent to services.
+ *
+ * @param {number} timestamp - The timestamp when the camera was off.
+ * @returns {AnyAction}
+ */
+export function faceLandmarkDetectionStopped(timestamp: number): AnyAction {
+    return {
+        type: FACE_LANDMARK_DETECTION_STOPPED,
+        timestamp
+    };
+}

react/features/face-landmarks/constants.ts

@@ -60,3 +60,8 @@ export const DETECTION_TYPES = {
  * Threshold for detection score of face.
  */
 export const FACE_DETECTION_SCORE_THRESHOLD = 0.75;
+
+/**
+ * Threshold for stopping detection after a certain number of consecutive errors have occurred.
+ */
+export const FACE_LANDMARK_DETECTION_ERROR_THRESHOLD = 4;

react/features/rtcstats/functions.ts

@@ -13,32 +13,33 @@ import {
 // @ts-ignore
 import { getLocalParticipant } from '../base/participants';
 
-// @ts-ignore
-import { toState } from '../base/redux';
+import { toState } from '../base/redux/functions';
 
 import RTCStats from './RTCStats';
 import logger from './logger';
+import { IStateful } from '../base/app/types';
+import { IStore } from '../app/types';
 
 /**
  * Checks whether rtcstats is enabled or not.
  *
- * @param {Function|Object} stateful - The redux store or {@code getState} function.
+ * @param {IStateful} stateful - The redux store or {@code getState} function.
  * @returns {boolean}
  */
-export function isRtcstatsEnabled(stateful: Function | Object) {
+export function isRtcstatsEnabled(stateful: IStateful) {
     const state = toState(stateful);
-    const config = state['features/base/config'];
+    const { analytics } = state['features/base/config'];
 
-    return config?.analytics?.rtcstatsEnabled ?? false;
+    return analytics?.rtcstatsEnabled ?? false;
 }
 
 /**
  * Can the rtcstats service send data.
  *
- * @param {Function|Object} stateful - The redux store or {@code getState} function.
+ * @param {IStateful} stateful - The redux store or {@code getState} function.
  * @returns {boolean}
  */
-export function canSendRtcstatsData(stateful: Function | Object) {
+export function canSendRtcstatsData(stateful: IStateful) {
     return isRtcstatsEnabled(stateful) && RTCStats.isInitialized();
 }
 
@@ -54,13 +55,12 @@ type Identity = {
 /**
  * Connects to the rtcstats service and sends the identity data.
  *
- * @param {Function} dispatch - The redux dispatch function.
- * @param  {Function|Object} stateful - The redux store or {@code getState} function.
+ * @param {IStore} store - Redux Store.
  * @param {Identity} identity - Identity data for the client.
  * @returns {void}
  */
-export function connectAndSendIdentity(dispatch: Function, stateful: Function | Object, identity: Identity) {
-    const state = toState(stateful);
+export function connectAndSendIdentity({ getState, dispatch }: IStore, identity: Identity) {
+    const state = getState();
 
     if (canSendRtcstatsData(state)) {
 
@@ -98,3 +98,20 @@ export function connectAndSendIdentity(dispatch: Function, stateful: Function |
     }
 
 }
+
+/**
+ * Checks if the faceLandmarks data can be sent to the rtcstats server.
+ *
+ * @param {IStateful} stateful - The redux store or {@code getState} function.
+ * @returns {boolean}
+ */
+export function canSendFaceLandmarksRtcstatsData(stateful: IStateful): boolean {
+    const state = toState(stateful);
+    const { faceLandmarks } = state['features/base/config'];
+
+    if (faceLandmarks?.enableRTCStats && canSendRtcstatsData(state)) {
+        return true;
+    }
+
+    return false;
+}

react/features/rtcstats/middleware.ts

@@ -1,4 +1,6 @@
-/* eslint-disable import/order */
+/* eslint-disable lines-around-comment */
+import { AnyAction } from 'redux';
+
 import { IStore } from '../app/types';
 import {
     E2E_RTT_CHANGED,
@@ -10,25 +12,21 @@ import {
     // @ts-ignore
 } from '../base/conference';
 import { LIB_WILL_INIT } from '../base/lib-jitsi-meet/actionTypes';
-
-// @ts-ignore
-import { DOMINANT_SPEAKER_CHANGED } from '../base/participants';
-
-// @ts-ignore
-import { MiddlewareRegistry } from '../base/redux';
-
-// @ts-ignore
-import { TRACK_ADDED, TRACK_UPDATED } from '../base/tracks';
-
-// @ts-ignore
+import { DOMINANT_SPEAKER_CHANGED } from '../base/participants/actionTypes';
+import MiddlewareRegistry from '../base/redux/MiddlewareRegistry';
+import { TRACK_ADDED, TRACK_UPDATED } from '../base/tracks/actionTypes';
 import { isInBreakoutRoom, getCurrentRoomId } from '../breakout-rooms/functions';
-
 // @ts-ignore
 import { extractFqnFromPath } from '../dynamic-branding/functions.any';
-import { ADD_FACE_EXPRESSION } from '../face-landmarks/actionTypes';
+import { ADD_FACE_EXPRESSION, FACE_LANDMARK_DETECTION_STOPPED } from '../face-landmarks/actionTypes';
 
 import RTCStats from './RTCStats';
-import { canSendRtcstatsData, connectAndSendIdentity, isRtcstatsEnabled } from './functions';
+import {
+    canSendFaceLandmarksRtcstatsData,
+    canSendRtcstatsData,
+    connectAndSendIdentity,
+    isRtcstatsEnabled
+} from './functions';
 import logger from './logger';
 
 /**
@@ -38,11 +36,11 @@ import logger from './logger';
  * @param {Store} store - The redux store.
  * @returns {Function}
  */
-MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: any) => {
-    const { dispatch, getState } = store;
+MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: AnyAction) => {
+    const { getState } = store;
     const state = getState();
     const config = state['features/base/config'];
-    const { analytics, faceLandmarks } = config;
+    const { analytics } = config;
 
     switch (action.type) {
     case LIB_WILL_INIT: {
@@ -78,8 +76,7 @@ MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: any)
     case CONFERENCE_JOINED: {
         if (isInBreakoutRoom(getState())) {
             connectAndSendIdentity(
-                dispatch,
-                state,
+                store,
                 {
                     isBreakoutRoom: true,
                     roomId: getCurrentRoomId(getState())
@@ -99,8 +96,7 @@ MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: any)
             const meetingUniqueId = conference?.getMeetingUniqueId();
 
             connectAndSendIdentity(
-                dispatch,
-                state,
+                store,
                 {
                     isBreakoutRoom: false,
                     meetingUniqueId
@@ -164,13 +160,14 @@ MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: any)
         }
         break;
     }
-    case ADD_FACE_EXPRESSION: {
-        if (canSendRtcstatsData(state) && faceLandmarks && faceLandmarks.enableRTCStats) {
+    case ADD_FACE_EXPRESSION:
+    case FACE_LANDMARK_DETECTION_STOPPED: {
+        if (canSendFaceLandmarksRtcstatsData(state)) {
             const { duration, faceExpression, timestamp } = action;
 
             RTCStats.sendFaceLandmarksData({
-                duration,
-                faceLandmarks: faceExpression,
+                duration: duration ?? 0,
+                faceLandmarks: faceExpression ?? 'detection-off',
                 timestamp
             });
         }