From ebb1b8d76b247f7bf6b4136c91eb282cc70e6979 Mon Sep 17 00:00:00 2001 From: Josh Brown Date: Thu, 17 Sep 2020 12:25:06 -0400 Subject: [PATCH] fix(background-blur) refactor to improve performance --- package-lock.json | 5 ++ package.json | 1 + .../blur/JitsiStreamBlurEffect.js | 83 +++++++++++++------ .../stream-effects/blur/TimerWorker.js | 28 +++---- 4 files changed, 79 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index d9e8a9d94..617a1a70b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15234,6 +15234,11 @@ "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz", "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==" }, + "stackblur-canvas": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/stackblur-canvas/-/stackblur-canvas-2.3.0.tgz", + "integrity": "sha512-3ZHJv+43D8YttgumssIxkfs3hBXW7XaMS5Ux65fOBhKDYMjbG5hF8Ey8a90RiiJ58aQnAhWbGilPzZ9rkIlWgQ==" + }, "stacktrace-parser": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/stacktrace-parser/-/stacktrace-parser-0.1.8.tgz", diff --git a/package.json b/package.json index 067ac774f..e038ac362 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "redux-thunk": "2.2.0", "rnnoise-wasm": "github:jitsi/rnnoise-wasm.git#566a16885897704d6e6d67a1d5ac5d39781db2af", "rtcstats": "github:jitsi/rtcstats#v6.2.0", + "stackblur-canvas": "2.3.0", "styled-components": "3.4.9", "util": "0.12.1", "uuid": "3.1.0", diff --git a/react/features/stream-effects/blur/JitsiStreamBlurEffect.js b/react/features/stream-effects/blur/JitsiStreamBlurEffect.js index 011bfe71e..2d7b65c42 100644 --- a/react/features/stream-effects/blur/JitsiStreamBlurEffect.js +++ b/react/features/stream-effects/blur/JitsiStreamBlurEffect.js @@ -1,11 +1,11 @@ // @flow -import * as bodyPix from '@tensorflow-models/body-pix'; +import * as StackBlur from 'stackblur-canvas'; import { - CLEAR_INTERVAL, - INTERVAL_TIMEOUT, - SET_INTERVAL, + CLEAR_TIMEOUT, + TIMEOUT_TICK, + SET_TIMEOUT, timerWorkerScript } from './TimerWorker'; @@ -17,6 +17,7 @@ import { export default class JitsiStreamBlurEffect { _bpModel: Object; _inputVideoElement: HTMLVideoElement; + _inputVideoCanvasElement: HTMLCanvasElement; _onMaskFrameTimer: Function; _maskFrameTimerWorker: Worker; _maskInProgress: boolean; @@ -43,6 +44,7 @@ export default class JitsiStreamBlurEffect { this._outputCanvasElement = document.createElement('canvas'); this._outputCanvasElement.getContext('2d'); this._inputVideoElement = document.createElement('video'); + this._inputVideoCanvasElement = document.createElement('canvas'); } /** @@ -53,10 +55,8 @@ export default class JitsiStreamBlurEffect { * @returns {void} */ async _onMaskFrameTimer(response: Object) { - if (response.data.id === INTERVAL_TIMEOUT) { - if (!this._maskInProgress) { - await this._renderMask(); - } + if (response.data.id === TIMEOUT_TICK) { + await this._renderMask(); } } @@ -67,20 +67,53 @@ export default class JitsiStreamBlurEffect { * @returns {void} */ async _renderMask() { - this._maskInProgress = true; - this._segmentationData = await this._bpModel.segmentPerson(this._inputVideoElement, { - internalResolution: 'medium', // resized to 0.5 times of the original resolution before inference - maxDetections: 1, // max. number of person poses to detect per image - segmentationThreshold: 0.7 // represents probability that a pixel belongs to a person - }); - this._maskInProgress = false; - bodyPix.drawBokehEffect( - this._outputCanvasElement, - this._inputVideoElement, - this._segmentationData, - 12, // Constant for background blur, integer values between 0-20 - 7 // Constant for edge blur, integer values between 0-20 + if (!this._maskInProgress) { + this._maskInProgress = true; + this._bpModel.segmentPerson(this._inputVideoElement, { + internalResolution: 'low', // resized to 0.5 times of the original resolution before inference + maxDetections: 1, // max. number of person poses to detect per image + segmentationThreshold: 0.7, // represents probability that a pixel belongs to a person + flipHorizontal: false, + scoreThreshold: 0.2 + }).then(data => { + this._segmentationData = data; + this._maskInProgress = false; + }); + } + const inputCanvasCtx = this._inputVideoCanvasElement.getContext('2d'); + + inputCanvasCtx.drawImage(this._inputVideoElement, 0, 0); + + const currentFrame = inputCanvasCtx.getImageData( + 0, + 0, + this._inputVideoCanvasElement.width, + this._inputVideoCanvasElement.height ); + + if (this._segmentationData) { + const blurData = new ImageData(currentFrame.data.slice(), currentFrame.width, currentFrame.height); + + StackBlur.imageDataRGB(blurData, 0, 0, currentFrame.width, currentFrame.height, 12); + + for (let x = 0; x < this._outputCanvasElement.width; x++) { + for (let y = 0; y < this._outputCanvasElement.height; y++) { + const n = (y * this._outputCanvasElement.width) + x; + + if (this._segmentationData.data[n] === 0) { + currentFrame.data[n * 4] = blurData.data[n * 4]; + currentFrame.data[(n * 4) + 1] = blurData.data[(n * 4) + 1]; + currentFrame.data[(n * 4) + 2] = blurData.data[(n * 4) + 2]; + currentFrame.data[(n * 4) + 3] = blurData.data[(n * 4) + 3]; + } + } + } + } + this._outputCanvasElement.getContext('2d').putImageData(currentFrame, 0, 0); + this._maskFrameTimerWorker.postMessage({ + id: SET_TIMEOUT, + timeMs: 1000 / 30 + }); } /** @@ -110,14 +143,16 @@ export default class JitsiStreamBlurEffect { this._outputCanvasElement.width = parseInt(width, 10); this._outputCanvasElement.height = parseInt(height, 10); + this._inputVideoCanvasElement.width = parseInt(width, 10); + this._inputVideoCanvasElement.height = parseInt(height, 10); this._inputVideoElement.width = parseInt(width, 10); this._inputVideoElement.height = parseInt(height, 10); this._inputVideoElement.autoplay = true; this._inputVideoElement.srcObject = stream; this._inputVideoElement.onloadeddata = () => { this._maskFrameTimerWorker.postMessage({ - id: SET_INTERVAL, - timeMs: 1000 / parseInt(frameRate, 10) + id: SET_TIMEOUT, + timeMs: 1000 / 30 }); }; @@ -131,7 +166,7 @@ export default class JitsiStreamBlurEffect { */ stopEffect() { this._maskFrameTimerWorker.postMessage({ - id: CLEAR_INTERVAL + id: CLEAR_TIMEOUT }); this._maskFrameTimerWorker.terminate(); diff --git a/react/features/stream-effects/blur/TimerWorker.js b/react/features/stream-effects/blur/TimerWorker.js index 4a0431c93..543be798d 100644 --- a/react/features/stream-effects/blur/TimerWorker.js +++ b/react/features/stream-effects/blur/TimerWorker.js @@ -1,34 +1,34 @@ /** - * SET_INTERVAL constant is used to set interval and it is set in + * SET_TIMEOUT constant is used to set interval and it is set in * the id property of the request.data property. timeMs property must * also be set. request.data example: * * { - * id: SET_INTERVAL, + * id: SET_TIMEOUT, * timeMs: 33 * } */ -export const SET_INTERVAL = 1; +export const SET_TIMEOUT = 1; /** - * CLEAR_INTERVAL constant is used to clear the interval and it is set in + * CLEAR_TIMEOUT constant is used to clear the interval and it is set in * the id property of the request.data property. * * { - * id: CLEAR_INTERVAL + * id: CLEAR_TIMEOUT * } */ -export const CLEAR_INTERVAL = 2; +export const CLEAR_TIMEOUT = 2; /** - * INTERVAL_TIMEOUT constant is used as response and it is set in the id property. + * TIMEOUT_TICK constant is used as response and it is set in the id property. * * { - * id: INTERVAL_TIMEOUT + * id: TIMEOUT_TICK * } */ -export const INTERVAL_TIMEOUT = 3; +export const TIMEOUT_TICK = 3; /** * The following code is needed as string to create a URL from a Blob. @@ -40,15 +40,15 @@ const code = ` onmessage = function(request) { switch (request.data.id) { - case ${SET_INTERVAL}: { - timer = setInterval(() => { - postMessage({ id: ${INTERVAL_TIMEOUT} }); + case ${SET_TIMEOUT}: { + timer = setTimeout(() => { + postMessage({ id: ${TIMEOUT_TICK} }); }, request.data.timeMs); break; } - case ${CLEAR_INTERVAL}: { + case ${CLEAR_TIMEOUT}: { if (timer) { - clearInterval(timer); + clearTimeout(timer); } break; }