fix(background-blur) refactor to improve performance

package-lock.json

@@ -15234,6 +15234,11 @@
       "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz",
       "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA=="
     },
+    "stackblur-canvas": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/stackblur-canvas/-/stackblur-canvas-2.3.0.tgz",
+      "integrity": "sha512-3ZHJv+43D8YttgumssIxkfs3hBXW7XaMS5Ux65fOBhKDYMjbG5hF8Ey8a90RiiJ58aQnAhWbGilPzZ9rkIlWgQ=="
+    },
     "stacktrace-parser": {
       "version": "0.1.8",
       "resolved": "https://registry.npmjs.org/stacktrace-parser/-/stacktrace-parser-0.1.8.tgz",

package.json

@@ -92,6 +92,7 @@
     "redux-thunk": "2.2.0",
     "rnnoise-wasm": "github:jitsi/rnnoise-wasm.git#566a16885897704d6e6d67a1d5ac5d39781db2af",
     "rtcstats": "github:jitsi/rtcstats#v6.2.0",
+    "stackblur-canvas": "2.3.0",
     "styled-components": "3.4.9",
     "util": "0.12.1",
     "uuid": "3.1.0",

react/features/stream-effects/blur/JitsiStreamBlurEffect.js

@@ -1,11 +1,11 @@
 // @flow
 
-import * as bodyPix from '@tensorflow-models/body-pix';
+import * as StackBlur from 'stackblur-canvas';
 
 import {
-    CLEAR_INTERVAL,
+    CLEAR_TIMEOUT,
-    INTERVAL_TIMEOUT,
+    TIMEOUT_TICK,
-    SET_INTERVAL,
+    SET_TIMEOUT,
     timerWorkerScript
 } from './TimerWorker';
 
@@ -17,6 +17,7 @@ import {
 export default class JitsiStreamBlurEffect {
     _bpModel: Object;
     _inputVideoElement: HTMLVideoElement;
+    _inputVideoCanvasElement: HTMLCanvasElement;
     _onMaskFrameTimer: Function;
     _maskFrameTimerWorker: Worker;
     _maskInProgress: boolean;
@@ -43,6 +44,7 @@ export default class JitsiStreamBlurEffect {
         this._outputCanvasElement = document.createElement('canvas');
         this._outputCanvasElement.getContext('2d');
         this._inputVideoElement = document.createElement('video');
+        this._inputVideoCanvasElement = document.createElement('canvas');
     }
 
     /**
@@ -53,10 +55,8 @@ export default class JitsiStreamBlurEffect {
      * @returns {void}
      */
     async _onMaskFrameTimer(response: Object) {
-        if (response.data.id === INTERVAL_TIMEOUT) {
+        if (response.data.id === TIMEOUT_TICK) {
-            if (!this._maskInProgress) {
+            await this._renderMask();
-                await this._renderMask();
-            }
         }
     }
 
@@ -67,20 +67,53 @@ export default class JitsiStreamBlurEffect {
      * @returns {void}
      */
     async _renderMask() {
-        this._maskInProgress = true;
+        if (!this._maskInProgress) {
-        this._segmentationData = await this._bpModel.segmentPerson(this._inputVideoElement, {
+            this._maskInProgress = true;
-            internalResolution: 'medium', // resized to 0.5 times of the original resolution before inference
+            this._bpModel.segmentPerson(this._inputVideoElement, {
-            maxDetections: 1, // max. number of person poses to detect per image
+                internalResolution: 'low', // resized to 0.5 times of the original resolution before inference
-            segmentationThreshold: 0.7 // represents probability that a pixel belongs to a person
+                maxDetections: 1, // max. number of person poses to detect per image
-        });
+                segmentationThreshold: 0.7, // represents probability that a pixel belongs to a person
-        this._maskInProgress = false;
+                flipHorizontal: false,
-        bodyPix.drawBokehEffect(
+                scoreThreshold: 0.2
-            this._outputCanvasElement,
+            }).then(data => {
-            this._inputVideoElement,
+                this._segmentationData = data;
-            this._segmentationData,
+                this._maskInProgress = false;
-            12, // Constant for background blur, integer values between 0-20
+            });
-            7 // Constant for edge blur, integer values between 0-20
+        }
+        const inputCanvasCtx = this._inputVideoCanvasElement.getContext('2d');
+
+        inputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
+
+        const currentFrame = inputCanvasCtx.getImageData(
+            0,
+            0,
+            this._inputVideoCanvasElement.width,
+            this._inputVideoCanvasElement.height
         );
+
+        if (this._segmentationData) {
+            const blurData = new ImageData(currentFrame.data.slice(), currentFrame.width, currentFrame.height);
+
+            StackBlur.imageDataRGB(blurData, 0, 0, currentFrame.width, currentFrame.height, 12);
+
+            for (let x = 0; x < this._outputCanvasElement.width; x++) {
+                for (let y = 0; y < this._outputCanvasElement.height; y++) {
+                    const n = (y * this._outputCanvasElement.width) + x;
+
+                    if (this._segmentationData.data[n] === 0) {
+                        currentFrame.data[n * 4] = blurData.data[n * 4];
+                        currentFrame.data[(n * 4) + 1] = blurData.data[(n * 4) + 1];
+                        currentFrame.data[(n * 4) + 2] = blurData.data[(n * 4) + 2];
+                        currentFrame.data[(n * 4) + 3] = blurData.data[(n * 4) + 3];
+                    }
+                }
+            }
+        }
+        this._outputCanvasElement.getContext('2d').putImageData(currentFrame, 0, 0);
+        this._maskFrameTimerWorker.postMessage({
+            id: SET_TIMEOUT,
+            timeMs: 1000 / 30
+        });
     }
 
     /**
@@ -110,14 +143,16 @@ export default class JitsiStreamBlurEffect {
 
         this._outputCanvasElement.width = parseInt(width, 10);
         this._outputCanvasElement.height = parseInt(height, 10);
+        this._inputVideoCanvasElement.width = parseInt(width, 10);
+        this._inputVideoCanvasElement.height = parseInt(height, 10);
         this._inputVideoElement.width = parseInt(width, 10);
         this._inputVideoElement.height = parseInt(height, 10);
         this._inputVideoElement.autoplay = true;
         this._inputVideoElement.srcObject = stream;
         this._inputVideoElement.onloadeddata = () => {
             this._maskFrameTimerWorker.postMessage({
-                id: SET_INTERVAL,
+                id: SET_TIMEOUT,
-                timeMs: 1000 / parseInt(frameRate, 10)
+                timeMs: 1000 / 30
             });
         };
 
@@ -131,7 +166,7 @@ export default class JitsiStreamBlurEffect {
      */
     stopEffect() {
         this._maskFrameTimerWorker.postMessage({
-            id: CLEAR_INTERVAL
+            id: CLEAR_TIMEOUT
         });
 
         this._maskFrameTimerWorker.terminate();

react/features/stream-effects/blur/TimerWorker.js

@@ -1,34 +1,34 @@
 
 /**
- * SET_INTERVAL constant is used to set interval and it is set in
+ * SET_TIMEOUT constant is used to set interval and it is set in
  * the id property of the request.data property. timeMs property must
  * also be set. request.data example:
  *
  * {
- *      id: SET_INTERVAL,
+ *      id: SET_TIMEOUT,
  *      timeMs: 33
  * }
  */
-export const SET_INTERVAL = 1;
+export const SET_TIMEOUT = 1;
 
 /**
- * CLEAR_INTERVAL constant is used to clear the interval and it is set in
+ * CLEAR_TIMEOUT constant is used to clear the interval and it is set in
  * the id property of the request.data property.
  *
  * {
- *      id: CLEAR_INTERVAL
+ *      id: CLEAR_TIMEOUT
  * }
  */
-export const CLEAR_INTERVAL = 2;
+export const CLEAR_TIMEOUT = 2;
 
 /**
- * INTERVAL_TIMEOUT constant is used as response and it is set in the id property.
+ * TIMEOUT_TICK constant is used as response and it is set in the id property.
  *
  * {
- *      id: INTERVAL_TIMEOUT
+ *      id: TIMEOUT_TICK
  * }
  */
-export const INTERVAL_TIMEOUT = 3;
+export const TIMEOUT_TICK = 3;
 
 /**
  * The following code is needed as string to create a URL from a Blob.
@@ -40,15 +40,15 @@ const code = `
 
     onmessage = function(request) {
         switch (request.data.id) {
-        case ${SET_INTERVAL}: {
+        case ${SET_TIMEOUT}: {
-            timer = setInterval(() => {
+            timer = setTimeout(() => {
-                postMessage({ id: ${INTERVAL_TIMEOUT} });
+                postMessage({ id: ${TIMEOUT_TICK} });
             }, request.data.timeMs);
             break;
         }
-        case ${CLEAR_INTERVAL}: {
+        case ${CLEAR_TIMEOUT}: {
             if (timer) {
-                clearInterval(timer);
+                clearTimeout(timer);
             }
             break;
         }