haskal
/
jiti-meet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(background-blur) refactor to improve performance

This commit is contained in:
Josh Brown 2020-09-17 12:25:06 -04:00 committed by GitHub
parent 42d559de93
commit ebb1b8d76b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 79 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
package-lock.json generated
View File

@ -15234,6 +15234,11 @@
"resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz",
"integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA=="
},
"stackblur-canvas": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/stackblur-canvas/-/stackblur-canvas-2.3.0.tgz",
"integrity": "sha512-3ZHJv+43D8YttgumssIxkfs3hBXW7XaMS5Ux65fOBhKDYMjbG5hF8Ey8a90RiiJ58aQnAhWbGilPzZ9rkIlWgQ=="
},
"stacktrace-parser": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/stacktrace-parser/-/stacktrace-parser-0.1.8.tgz",

1
package.json
View File

@ -92,6 +92,7 @@
"redux-thunk": "2.2.0",
"rnnoise-wasm": "github:jitsi/rnnoise-wasm.git#566a16885897704d6e6d67a1d5ac5d39781db2af",
"rtcstats": "github:jitsi/rtcstats#v6.2.0",
"stackblur-canvas": "2.3.0",
"styled-components": "3.4.9",
"util": "0.12.1",
"uuid": "3.1.0",

75
react/features/stream-effects/blur/JitsiStreamBlurEffect.js
View File

@ -1,11 +1,11 @@
// @flow
import * as bodyPix from '@tensorflow-models/body-pix';
import * as StackBlur from 'stackblur-canvas';
import {
CLEAR_INTERVAL,
INTERVAL_TIMEOUT,
SET_INTERVAL,
CLEAR_TIMEOUT,
TIMEOUT_TICK,
SET_TIMEOUT,
timerWorkerScript
} from './TimerWorker';
@ -17,6 +17,7 @@ import {
export default class JitsiStreamBlurEffect {
_bpModel: Object;
_inputVideoElement: HTMLVideoElement;
_inputVideoCanvasElement: HTMLCanvasElement;
_onMaskFrameTimer: Function;
_maskFrameTimerWorker: Worker;
_maskInProgress: boolean;
@ -43,6 +44,7 @@ export default class JitsiStreamBlurEffect {
this._outputCanvasElement = document.createElement('canvas');
this._outputCanvasElement.getContext('2d');
this._inputVideoElement = document.createElement('video');
this._inputVideoCanvasElement = document.createElement('canvas');
}
/**
@ -53,12 +55,10 @@ export default class JitsiStreamBlurEffect {
* @returns {void}
*/
async _onMaskFrameTimer(response: Object) {
if (response.data.id === INTERVAL_TIMEOUT) {
if (!this._maskInProgress) {
if (response.data.id === TIMEOUT_TICK) {
await this._renderMask();
}
}
}
/**
* Loop function to render the background mask.
@ -67,20 +67,53 @@ export default class JitsiStreamBlurEffect {
* @returns {void}
*/
async _renderMask() {
if (!this._maskInProgress) {
this._maskInProgress = true;
this._segmentationData = await this._bpModel.segmentPerson(this._inputVideoElement, {
internalResolution: 'medium', // resized to 0.5 times of the original resolution before inference
this._bpModel.segmentPerson(this._inputVideoElement, {
internalResolution: 'low', // resized to 0.5 times of the original resolution before inference
maxDetections: 1, // max. number of person poses to detect per image
segmentationThreshold: 0.7 // represents probability that a pixel belongs to a person
});
segmentationThreshold: 0.7, // represents probability that a pixel belongs to a person
flipHorizontal: false,
scoreThreshold: 0.2
}).then(data => {
this._segmentationData = data;
this._maskInProgress = false;
bodyPix.drawBokehEffect(
this._outputCanvasElement,
this._inputVideoElement,
this._segmentationData,
12, // Constant for background blur, integer values between 0-20
7 // Constant for edge blur, integer values between 0-20
});
}
const inputCanvasCtx = this._inputVideoCanvasElement.getContext('2d');
inputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
const currentFrame = inputCanvasCtx.getImageData(
0,
0,
this._inputVideoCanvasElement.width,
this._inputVideoCanvasElement.height
);
if (this._segmentationData) {
const blurData = new ImageData(currentFrame.data.slice(), currentFrame.width, currentFrame.height);
StackBlur.imageDataRGB(blurData, 0, 0, currentFrame.width, currentFrame.height, 12);
for (let x = 0; x < this._outputCanvasElement.width; x++) {
for (let y = 0; y < this._outputCanvasElement.height; y++) {
const n = (y * this._outputCanvasElement.width) + x;
if (this._segmentationData.data[n] === 0) {
currentFrame.data[n * 4] = blurData.data[n * 4];
currentFrame.data[(n * 4) + 1] = blurData.data[(n * 4) + 1];
currentFrame.data[(n * 4) + 2] = blurData.data[(n * 4) + 2];
currentFrame.data[(n * 4) + 3] = blurData.data[(n * 4) + 3];
}
}
}
}
this._outputCanvasElement.getContext('2d').putImageData(currentFrame, 0, 0);
this._maskFrameTimerWorker.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30
});
}
/**
@ -110,14 +143,16 @@ export default class JitsiStreamBlurEffect {
this._outputCanvasElement.width = parseInt(width, 10);
this._outputCanvasElement.height = parseInt(height, 10);
this._inputVideoCanvasElement.width = parseInt(width, 10);
this._inputVideoCanvasElement.height = parseInt(height, 10);
this._inputVideoElement.width = parseInt(width, 10);
this._inputVideoElement.height = parseInt(height, 10);
this._inputVideoElement.autoplay = true;
this._inputVideoElement.srcObject = stream;
this._inputVideoElement.onloadeddata = () => {
this._maskFrameTimerWorker.postMessage({
id: SET_INTERVAL,
timeMs: 1000 / parseInt(frameRate, 10)
id: SET_TIMEOUT,
timeMs: 1000 / 30
});
};
@ -131,7 +166,7 @@ export default class JitsiStreamBlurEffect {
*/
stopEffect() {
this._maskFrameTimerWorker.postMessage({
id: CLEAR_INTERVAL
id: CLEAR_TIMEOUT
});
this._maskFrameTimerWorker.terminate();

28
react/features/stream-effects/blur/TimerWorker.js
View File

@ -1,34 +1,34 @@
/**
* SET_INTERVAL constant is used to set interval and it is set in
* SET_TIMEOUT constant is used to set interval and it is set in
* the id property of the request.data property. timeMs property must
* also be set. request.data example:
*
* {
* id: SET_INTERVAL,
* id: SET_TIMEOUT,
* timeMs: 33
* }
*/
export const SET_INTERVAL = 1;
export const SET_TIMEOUT = 1;
/**
* CLEAR_INTERVAL constant is used to clear the interval and it is set in
* CLEAR_TIMEOUT constant is used to clear the interval and it is set in
* the id property of the request.data property.
*
* {
* id: CLEAR_INTERVAL
* id: CLEAR_TIMEOUT
* }
*/
export const CLEAR_INTERVAL = 2;
export const CLEAR_TIMEOUT = 2;
/**
* INTERVAL_TIMEOUT constant is used as response and it is set in the id property.
* TIMEOUT_TICK constant is used as response and it is set in the id property.
*
* {
* id: INTERVAL_TIMEOUT
* id: TIMEOUT_TICK
* }
*/
export const INTERVAL_TIMEOUT = 3;
export const TIMEOUT_TICK = 3;
/**
* The following code is needed as string to create a URL from a Blob.
@ -40,15 +40,15 @@ const code = `
onmessage = function(request) {
switch (request.data.id) {
case ${SET_INTERVAL}: {
timer = setInterval(() => {
postMessage({ id: ${INTERVAL_TIMEOUT} });
case ${SET_TIMEOUT}: {
timer = setTimeout(() => {
postMessage({ id: ${TIMEOUT_TICK} });
}, request.data.timeMs);
break;
}
case ${CLEAR_INTERVAL}: {
case ${CLEAR_TIMEOUT}: {
if (timer) {
clearInterval(timer);
clearTimeout(timer);
}
break;
}